62#include "llvm/IR/IntrinsicsAArch64.h"
97#define DEBUG_TYPE "aarch64-lower"
108 cl::desc(
"Allow AArch64 Local Dynamic TLS code generation"),
113 cl::desc(
"Enable AArch64 logical imm instruction "
123 cl::desc(
"Combine extends of AArch64 masked "
124 "gather intrinsics"),
156 switch (EC.getKnownMinValue()) {
172 "Expected scalable predicate vector type!");
194 "Expected legal vector type!");
288 if (Subtarget->
hasSVE()) {
316 if (useSVEForFixedLengthVectorVT(VT))
320 if (useSVEForFixedLengthVectorVT(VT))
761#define LCALLNAMES(A, B, N) \
762 setLibcallName(A##N##_RELAX, #B #N "_relax"); \
763 setLibcallName(A##N##_ACQ, #B #N "_acq"); \
764 setLibcallName(A##N##_REL, #B #N "_rel"); \
765 setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
766#define LCALLNAME4(A, B) \
767 LCALLNAMES(A, B, 1) \
768 LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
769#define LCALLNAME5(A, B) \
770 LCALLNAMES(A, B, 1) \
771 LCALLNAMES(A, B, 2) \
772 LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
1150 if (Subtarget->
hasSVE()) {
1314 if (useSVEForFixedLengthVectorVT(VT))
1315 addTypeForFixedLengthSVE(VT);
1317 if (useSVEForFixedLengthVectorVT(VT))
1318 addTypeForFixedLengthSVE(VT);
1400void AArch64TargetLowering::addTypeForNEON(
MVT VT) {
1469 for (
unsigned Opcode :
1482void AArch64TargetLowering::addTypeForFixedLengthSVE(
MVT VT) {
1600void AArch64TargetLowering::addDRTypeForNEON(
MVT VT) {
1605void AArch64TargetLowering::addQRTypeForNEON(
MVT VT) {
1624 uint64_t Mask = ((uint64_t)(-1LL) >> (64 -
Size)),
OrigMask = Mask;
1628 if (Imm == 0 || Imm == Mask ||
1632 unsigned EltSize =
Size;
1682 while (EltSize <
Size) {
1689 "demanded bits should never be altered");
1693 EVT VT = Op.getValueType();
1700 New =
TLO.DAG.getNode(Op.getOpcode(),
DL, VT, Op.getOperand(0),
1711 return TLO.CombineTo(Op, New);
1724 EVT VT = Op.getValueType();
1730 "i32 or i64 is expected after legalization.");
1737 switch (Op.getOpcode()) {
1741 NewOpc =
Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1744 NewOpc =
Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1747 NewOpc =
Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1753 uint64_t Imm =
C->getZExtValue();
1762 switch (Op.getOpcode()) {
1785 case Intrinsic::aarch64_ldaxr:
1786 case Intrinsic::aarch64_ldxr: {
1802 case Intrinsic::aarch64_neon_umaxv:
1803 case Intrinsic::aarch64_neon_uminv: {
1808 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1865 Ty.getSizeInBytes() != 16 ||
1888#define MAKE_CASE(V) \
2197 Register DestReg =
MI.getOperand(0).getReg();
2200 unsigned CondCode =
MI.getOperand(3).getImm();
2232 MI.eraseFromParent();
2240 "SEH does not use catchret!");
2246 switch (
MI.getOpcode()) {
2253 case AArch64::F128CSEL:
2256 case TargetOpcode::STACKMAP:
2257 case TargetOpcode::PATCHPOINT:
2258 case TargetOpcode::STATEPOINT:
2261 case AArch64::CATCHRET:
2285 N =
N->getOperand(0).getNode();
2293 auto Opnd0 =
N->getOperand(0);
2296 return (
CINT &&
CINT->isNullValue()) || (CFP && CFP->isZero());
2455 bool IsLegal = (
C >> 12 == 0) || ((
C & 0xFFFULL) == 0 &&
C >> 24 == 0);
2457 <<
" legal: " << (
IsLegal ?
"yes\n" :
"no\n"));
2479 EVT VT = LHS.getValueType();
2481 assert(VT !=
MVT::f16 &&
"Lowering of strict fp16 not yet implemented");
2489 EVT VT = LHS.getValueType();
2509 if (
isCMN(RHS, CC)) {
2512 RHS = RHS.getOperand(1);
2513 }
else if (
isCMN(LHS, CC)) {
2517 LHS = LHS.getOperand(1);
2532 return LHS.getValue(1);
2598 unsigned Opcode = 0;
2602 if (LHS.getValueType().isFloatingPoint()) {
2609 }
else if (RHS.getOpcode() ==
ISD::SUB) {
2614 RHS = RHS.getOperand(1);
2643 unsigned Depth = 0) {
2712 bool isInteger = LHS.getValueType().isInteger();
2714 CC = getSetCCInverse(CC, LHS.getValueType());
2720 assert(LHS.getValueType().isFloatingPoint());
2752 assert(
ValidL &&
"Valid conjunction/disjunction tree");
2759 assert(
ValidR &&
"Valid conjunction/disjunction tree");
2791 assert(Opcode ==
ISD::AND &&
"Valid conjunction/disjunction tree");
2792 assert(!Negate &&
"Valid conjunction/disjunction tree");
2836 uint64_t Mask =
MaskCst->getZExtValue();
2837 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
2843 if (!Op.hasOneUse())
2849 unsigned Opc = Op.getOpcode();
2852 uint64_t Shift =
ShiftCst->getZExtValue();
2854 return (Shift <= 4) ? 2 : 1;
2855 EVT VT = Op.getValueType();
2867 EVT VT = RHS.getValueType();
2868 uint64_t
C =
RHSC->getZExtValue();
2876 if ((VT ==
MVT::i32 &&
C != 0x80000000 &&
2878 (VT ==
MVT::i64 &&
C != 0x80000000ULL &&
2965 LHS.getNode()->hasNUsesOfValue(1, 0)) {
2972 RHS.getValueType()),
2978 if (!Cmp && (
RHSC->isNullValue() ||
RHSC->isOne())) {
2994static std::pair<SDValue, SDValue>
2997 "Unsupported value type");
3000 SDValue LHS = Op.getOperand(0);
3001 SDValue RHS = Op.getOperand(1);
3003 switch (Op.getOpcode()) {
3026 bool IsSigned = Op.getOpcode() ==
ISD::SMULO;
3027 if (Op.getValueType() ==
MVT::i32) {
3050 assert(Op.getValueType() ==
MVT::i64 &&
"Expected an i64 value type");
3085 if (useSVEForFixedLengthVectorVT(
Op.getValueType()))
3086 return LowerToScalableOp(Op, DAG);
3145 if (
CTVal->isAllOnesValue() &&
CFVal->isNullValue()) {
3152 if (
CTVal->isNullValue() &&
CFVal->isAllOnesValue()) {
3168 EVT VT = Op.getValueType();
3178 switch (Op.getOpcode()) {
3198 return DAG.
getNode(Opc,
SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
3199 return DAG.
getNode(Opc,
SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
3252 unsigned PrfOp = (IsWrite << 4) |
3262 EVT VT =
Op.getValueType();
3266 if (useSVEForFixedLengthVectorVT(VT))
3267 return LowerFixedLengthFPExtendToSVE(Op, DAG);
3275 if (
Op.getValueType().isScalableVector())
3282 if (useSVEForFixedLengthVectorVT(
SrcVT))
3283 return LowerFixedLengthFPRoundToSVE(Op, DAG);
3287 if (useSVEForFixedLengthVectorVT(
SrcVT))
3302 EVT InVT =
Op.getOperand(0).getValueType();
3303 EVT VT =
Op.getValueType();
3309 return LowerToPredicatedOp(Op, DAG, Opcode);
3312 if (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(
InVT))
3313 return LowerFixedLengthFPToIntToSVE(Op, DAG);
3323 Op.getOpcode(), dl,
Op.getValueType(),
3332 DAG.
getNode(
Op.getOpcode(), dl,
InVT.changeVectorElementTypeToInteger(),
3343 return DAG.
getNode(
Op.getOpcode(), dl, VT, Ext);
3355 if (
SrcVal.getValueType().isVector())
3356 return LowerVectorFP_TO_INT(Op, DAG);
3360 assert(!
IsStrict &&
"Lowering of strict fp16 not yet implemented");
3363 Op.getOpcode(), dl,
Op.getValueType(),
3390 if (
SrcVT.isVector())
3418 EVT VT =
Op.getValueType();
3422 unsigned Opc =
Op.getOpcode();
3431 return DAG.
getNode(Opc, dl, VT, In);
3436 return LowerToPredicatedOp(Op, DAG, Opcode);
3439 if (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(
InVT))
3440 return LowerFixedLengthIntToFPToSVE(Op, DAG);
3447 InVT.getVectorNumElements());
3456 return DAG.
getNode(Opc, dl, VT, In);
3464 if (
Op.getValueType().isVector())
3465 return LowerVectorINT_TO_FP(Op, DAG);
3473 assert(!
IsStrict &&
"Lowering of strict fp16 not yet implemented");
3498 EVT ArgVT =
Arg.getValueType();
3506 Entry.IsSExt =
false;
3507 Entry.IsZExt =
false;
3508 Args.push_back(Entry);
3511 : RTLIB::SINCOS_STRET_F32;
3531 EVT ArgVT =
Op.getOperand(0).getValueType();
3533 if (useSVEForFixedLengthVectorVT(
OpVT))
3534 return LowerFixedLengthBitcastToSVE(Op, DAG);
3536 if (
OpVT.isScalableVector()) {
3539 "Expected int->fp bitcast!");
3545 return getSVESafeBitCast(
OpVT,
Op.getOperand(0), DAG);
3563 if (
OrigVT.getSizeInBits() >= 64)
3566 assert(
OrigVT.isSimple() &&
"Expecting a simple value type");
3598 EVT VT =
N->getValueType(0);
3631 EVT VT =
N->getValueType(0);
3660 unsigned Opcode =
N->getOpcode();
3663 SDNode *
N1 =
N->getOperand(1).getNode();
3671 unsigned Opcode =
N->getOpcode();
3674 SDNode *
N1 =
N->getOperand(1).getNode();
3734 Chain =
FPCR.getValue(1);
3749 EVT VT =
Op.getValueType();
3760 "unexpected type for custom-lowering ISD::MUL");
3761 SDNode *N0 =
Op.getOperand(0).getNode();
3808 "unexpected types for extended operands to VMULL");
3846 switch (
InOp.getOpcode()) {
3850 if (
InOp.getConstantOperandVal(0) == Intrinsic::aarch64_sve_ptrue)
3861SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(
SDValue Op,
3867 case Intrinsic::thread_pointer: {
3871 case Intrinsic::aarch64_neon_abs: {
3872 EVT Ty =
Op.getValueType();
3878 }
else if (Ty.isVector() && Ty.isInteger() &&
isTypeLegal(Ty)) {
3884 case Intrinsic::aarch64_neon_smax:
3886 Op.getOperand(1),
Op.getOperand(2));
3887 case Intrinsic::aarch64_neon_umax:
3889 Op.getOperand(1),
Op.getOperand(2));
3890 case Intrinsic::aarch64_neon_smin:
3892 Op.getOperand(1),
Op.getOperand(2));
3893 case Intrinsic::aarch64_neon_umin:
3895 Op.getOperand(1),
Op.getOperand(2));
3897 case Intrinsic::aarch64_sve_sunpkhi:
3900 case Intrinsic::aarch64_sve_sunpklo:
3903 case Intrinsic::aarch64_sve_uunpkhi:
3906 case Intrinsic::aarch64_sve_uunpklo:
3909 case Intrinsic::aarch64_sve_clasta_n:
3911 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
3912 case Intrinsic::aarch64_sve_clastb_n:
3914 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
3915 case Intrinsic::aarch64_sve_lasta:
3917 Op.getOperand(1),
Op.getOperand(2));
3918 case Intrinsic::aarch64_sve_lastb:
3920 Op.getOperand(1),
Op.getOperand(2));
3921 case Intrinsic::aarch64_sve_rev:
3924 case Intrinsic::aarch64_sve_tbl:
3926 Op.getOperand(1),
Op.getOperand(2));
3927 case Intrinsic::aarch64_sve_trn1:
3929 Op.getOperand(1),
Op.getOperand(2));
3930 case Intrinsic::aarch64_sve_trn2:
3932 Op.getOperand(1),
Op.getOperand(2));
3933 case Intrinsic::aarch64_sve_uzp1:
3935 Op.getOperand(1),
Op.getOperand(2));
3936 case Intrinsic::aarch64_sve_uzp2:
3938 Op.getOperand(1),
Op.getOperand(2));
3939 case Intrinsic::aarch64_sve_zip1:
3941 Op.getOperand(1),
Op.getOperand(2));
3942 case Intrinsic::aarch64_sve_zip2:
3944 Op.getOperand(1),
Op.getOperand(2));
3945 case Intrinsic::aarch64_sve_splice:
3947 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
3948 case Intrinsic::aarch64_sve_ptrue:
3951 case Intrinsic::aarch64_sve_clz:
3953 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3954 case Intrinsic::aarch64_sve_cnt: {
3957 if (
Data.getValueType().isFloatingPoint())
3960 Op.getOperand(2),
Data,
Op.getOperand(1));
3962 case Intrinsic::aarch64_sve_dupq_lane:
3963 return LowerDUPQLane(Op, DAG);
3964 case Intrinsic::aarch64_sve_convert_from_svbool:
3967 case Intrinsic::aarch64_sve_convert_to_svbool:
3969 case Intrinsic::aarch64_sve_fneg:
3971 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3972 case Intrinsic::aarch64_sve_frintp:
3974 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3975 case Intrinsic::aarch64_sve_frintm:
3977 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3978 case Intrinsic::aarch64_sve_frinti:
3980 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3981 case Intrinsic::aarch64_sve_frintx:
3983 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3984 case Intrinsic::aarch64_sve_frinta:
3986 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3987 case Intrinsic::aarch64_sve_frintn:
3989 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3990 case Intrinsic::aarch64_sve_frintz:
3992 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3993 case Intrinsic::aarch64_sve_ucvtf:
3995 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3997 case Intrinsic::aarch64_sve_scvtf:
3999 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
4001 case Intrinsic::aarch64_sve_fcvtzu:
4003 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
4005 case Intrinsic::aarch64_sve_fcvtzs:
4007 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
4009 case Intrinsic::aarch64_sve_fsqrt:
4011 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
4012 case Intrinsic::aarch64_sve_frecpx:
4014 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
4015 case Intrinsic::aarch64_sve_fabs:
4017 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
4018 case Intrinsic::aarch64_sve_abs:
4020 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
4021 case Intrinsic::aarch64_sve_neg:
4023 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
4024 case Intrinsic::aarch64_sve_insr: {
4031 Op.getOperand(1), Scalar);
4033 case Intrinsic::aarch64_sve_rbit:
4035 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
4037 case Intrinsic::aarch64_sve_revb:
4039 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
4040 case Intrinsic::aarch64_sve_sxtb:
4043 Op.getOperand(2),
Op.getOperand(3),
4046 case Intrinsic::aarch64_sve_sxth:
4049 Op.getOperand(2),
Op.getOperand(3),
4052 case Intrinsic::aarch64_sve_sxtw:
4055 Op.getOperand(2),
Op.getOperand(3),
4058 case Intrinsic::aarch64_sve_uxtb:
4061 Op.getOperand(2),
Op.getOperand(3),
4064 case Intrinsic::aarch64_sve_uxth:
4067 Op.getOperand(2),
Op.getOperand(3),
4070 case Intrinsic::aarch64_sve_uxtw:
4073 Op.getOperand(2),
Op.getOperand(3),
4077 case Intrinsic::localaddress: {
4080 unsigned Reg =
RegInfo->getLocalAddressRegister(MF);
4082 Op.getSimpleValueType());
4085 case Intrinsic::eh_recoverfp: {
4095 "llvm.eh.recoverfp must take a function as the first argument");
4099 case Intrinsic::aarch64_neon_vsri:
4100 case Intrinsic::aarch64_neon_vsli: {
4101 EVT Ty =
Op.getValueType();
4106 assert(
Op.getConstantOperandVal(3) <= Ty.getScalarSizeInBits());
4110 return DAG.
getNode(Opcode, dl, Ty,
Op.getOperand(1),
Op.getOperand(2),
4114 case Intrinsic::aarch64_neon_srhadd:
4115 case Intrinsic::aarch64_neon_urhadd:
4116 case Intrinsic::aarch64_neon_shadd:
4117 case Intrinsic::aarch64_neon_uhadd: {
4119 IntNo == Intrinsic::aarch64_neon_shadd);
4121 IntNo == Intrinsic::aarch64_neon_urhadd);
4125 return DAG.
getNode(Opcode, dl,
Op.getValueType(),
Op.getOperand(1),
4128 case Intrinsic::aarch64_neon_sabd:
4129 case Intrinsic::aarch64_neon_uabd: {
4130 unsigned Opcode =
IntNo == Intrinsic::aarch64_neon_uabd ?
ISD::ABDU
4132 return DAG.
getNode(Opcode, dl,
Op.getValueType(),
Op.getOperand(1),
4135 case Intrinsic::aarch64_neon_uaddlp: {
4137 return DAG.
getNode(Opcode, dl,
Op.getValueType(),
Op.getOperand(1));
4139 case Intrinsic::aarch64_neon_sdot:
4140 case Intrinsic::aarch64_neon_udot:
4141 case Intrinsic::aarch64_sve_sdot:
4142 case Intrinsic::aarch64_sve_udot: {
4143 unsigned Opcode = (
IntNo == Intrinsic::aarch64_neon_udot ||
4144 IntNo == Intrinsic::aarch64_sve_udot)
4147 return DAG.
getNode(Opcode, dl,
Op.getValueType(),
Op.getOperand(1),
4148 Op.getOperand(2),
Op.getOperand(3));
4153bool AArch64TargetLowering::shouldExtendGSIndex(
EVT VT,
EVT &EltTy)
const {
4162bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(
EVT VT)
const {
4171bool AArch64TargetLowering::isVectorLoadExtDesirable(
SDValue ExtVal)
const {
4172 return ExtVal.getValueType().isScalableVector();
4176 std::map<std::tuple<bool, bool, bool>,
unsigned> AddrModes = {
4177 {std::make_tuple(
false,
false,
false),
4179 {std::make_tuple(
false,
false,
true),
4181 {std::make_tuple(
false,
true,
false),
4183 {std::make_tuple(
false,
true,
true),
4185 {std::make_tuple(
true,
false,
false),
4187 {std::make_tuple(
true,
false,
true),
4189 {std::make_tuple(
true,
true,
false),
4191 {std::make_tuple(
true,
true,
true),
4195 return AddrModes.find(Key)->second;
4199 std::map<std::tuple<bool, bool, bool>,
unsigned> AddrModes = {
4200 {std::make_tuple(
false,
false,
false),
4202 {std::make_tuple(
false,
false,
true),
4204 {std::make_tuple(
false,
true,
false),
4206 {std::make_tuple(
false,
true,
true),
4208 {std::make_tuple(
true,
false,
false),
4210 {std::make_tuple(
true,
false,
true),
4212 {std::make_tuple(
true,
true,
false),
4214 {std::make_tuple(
true,
true,
true),
4218 return AddrModes.find(Key)->second;
4244 unsigned Opcode = Index.getOpcode();
4253 if (!Mask || Mask->getZExtValue() != 0xFFFFFFFF)
4289 Index = Index->getOperand(0);
4310 Index = Index->getOperand(0);
4316 BasePtr = Index->getOperand(0);
4324 assert(
MGT &&
"Can only custom lower gather load nodes");
4342 Index.getSimpleValueType().getVectorElementType() ==
MVT::i32;
4356 "Cannot lower when not using SVE for fixed vectors");
4357 if (
MemVT.getScalarSizeInBits() <=
IndexVT.getScalarSizeInBits()) {
4385 Index = Index.getOperand(0);
4395 if (Index.getSimpleValueType().isFixedLengthVector())
4397 if (
BasePtr.getSimpleValueType().isFixedLengthVector())
4404 Chain =
Result.getValue(1);
4420 Result = getSVESafeBitCast(VT, Result, DAG);
4430 assert(
MSC &&
"Can only custom lower scatter store nodes");
4447 Index.getSimpleValueType().getVectorElementType() ==
MVT::i32;
4461 "Cannot lower when not using SVE for fixed vectors");
4462 if (
MemVT.getScalarSizeInBits() <=
IndexVT.getScalarSizeInBits()) {
4495 if (Index.getSimpleValueType().isFixedLengthVector())
4497 if (
BasePtr.getSimpleValueType().isFixedLengthVector())
4503 return DAG.
getNode(Opcode,
DL, VTs, Ops);
4509 assert(
LoadNode &&
"Expected custom lowering of a masked load node");
4510 EVT VT =
Op->getValueType(0);
4512 if (useSVEForFixedLengthVectorVT(VT,
true))
4513 return LowerFixedLengthVectorMLoadToSVE(Op, DAG);
4550 {Undef, Undef, Undef, Undef});
4561 ST->getBasePtr(), ST->getMemOperand());
4579 if (useSVEForFixedLengthVectorVT(VT,
true))
4580 return LowerFixedLengthVectorStoreToSVE(Op, DAG);
4582 unsigned AS =
StoreNode->getAddressSpace();
4584 if (Alignment <
MemVT.getStoreSize() &&
4599 if (
StoreNode->isNonTemporal() &&
MemVT.getSizeInBits() == 256u &&
4601 ((
MemVT.getScalarSizeInBits() == 8u ||
4602 MemVT.getScalarSizeInBits() == 16u ||
4603 MemVT.getScalarSizeInBits() == 32u ||
4604 MemVT.getScalarSizeInBits() == 64u))) {
4616 {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4630 {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4639 for (
unsigned i = 0;
i < 8;
i++) {
4664 for (
unsigned i = 0;
i < 8;
i++) {
4670 Ops.push_back(Part);
4678 EVT VT =
Op->getValueType(0);
4708 MVT VT =
Op.getSimpleValueType();
4730 switch (Op.getOpcode()) {
4735 return LowerBITCAST(Op, DAG);
4737 return LowerGlobalAddress(Op, DAG);
4739 return LowerGlobalTLSAddress(Op, DAG);
4743 return LowerSETCC(Op, DAG);
4745 return LowerBR_CC(Op, DAG);
4747 return LowerSELECT(Op, DAG);
4749 return LowerSELECT_CC(Op, DAG);
4751 return LowerJumpTable(Op, DAG);
4753 return LowerBR_JT(Op, DAG);
4755 return LowerConstantPool(Op, DAG);
4757 return LowerBlockAddress(Op, DAG);
4759 return LowerVASTART(Op, DAG);
4761 return LowerVACOPY(Op, DAG);
4763 return LowerVAARG(Op, DAG);
4808 return LowerFP_ROUND(Op, DAG);
4810 return LowerFP_EXTEND(Op, DAG);
4812 return LowerFRAMEADDR(Op, DAG);
4814 return LowerSPONENTRY(Op, DAG);
4816 return LowerRETURNADDR(Op, DAG);
4818 return LowerADDROFRETURNADDR(Op, DAG);
4820 return LowerCONCAT_VECTORS(Op, DAG);
4822 return LowerINSERT_VECTOR_ELT(Op, DAG);
4824 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
4826 return LowerBUILD_VECTOR(Op, DAG);
4828 return LowerVECTOR_SHUFFLE(Op, DAG);
4830 return LowerSPLAT_VECTOR(Op, DAG);
4832 return LowerEXTRACT_SUBVECTOR(Op, DAG);
4834 return LowerINSERT_SUBVECTOR(Op, DAG);
4837 return LowerDIV(Op, DAG);
4853 return LowerVectorSRA_SRL_SHL(Op, DAG);
4857 return LowerShiftParts(Op, DAG);
4859 return LowerCTPOP(Op, DAG);
4861 return LowerFCOPYSIGN(Op, DAG);
4863 return LowerVectorOR(Op, DAG);
4865 return LowerXOR(Op, DAG);
4872 return LowerINT_TO_FP(Op, DAG);
4877 return LowerFP_TO_INT(Op, DAG);
4880 return LowerFP_TO_INT_SAT(Op, DAG);
4882 return LowerFSINCOS(Op, DAG);
4884 return LowerFLT_ROUNDS_(Op, DAG);
4886 return LowerSET_ROUNDING(Op, DAG);
4888 return LowerMUL(Op, DAG);
4896 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
4898 return LowerSTORE(Op, DAG);
4900 return LowerFixedLengthVectorMStoreToSVE(Op, DAG);
4902 return LowerMGATHER(Op, DAG);
4904 return LowerMSCATTER(Op, DAG);
4906 return LowerVECREDUCE_SEQ_FADD(Op, DAG);
4918 return LowerVECREDUCE(Op, DAG);
4920 return LowerATOMIC_LOAD_SUB(Op, DAG);
4922 return LowerATOMIC_LOAD_AND(Op, DAG);
4924 return LowerDYNAMIC_STACKALLOC(Op, DAG);
4926 return LowerVSCALE(Op, DAG);
4930 return LowerFixedLengthVectorIntExtendToSVE(Op, DAG);
4939 return LowerToPredicatedOp(Op, DAG,
4943 return LowerTRUNCATE(Op, DAG);
4945 return LowerMLOAD(Op, DAG);
4947 if (useSVEForFixedLengthVectorVT(Op.getValueType()))
4948 return LowerFixedLengthVectorLoadToSVE(Op, DAG);
4949 return LowerLOAD(Op, DAG);
4953 return LowerToScalableOp(Op, DAG);
4965 return LowerFixedLengthVectorSelectToSVE(Op, DAG);
4967 return LowerABS(Op, DAG);
4969 return LowerBitreverse(Op, DAG);
4976 return LowerCTTZ(Op, DAG);
4978 return LowerVECTOR_SPLICE(Op, DAG);
4986bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
5037 bool IsVarArg)
const {
5076SDValue AArch64TargetLowering::LowerFormalArguments(
5096 unsigned NumArgs = Ins.size();
5099 for (
unsigned i = 0;
i != NumArgs; ++
i) {
5100 MVT ValVT = Ins[
i].VT;
5101 if (Ins[
i].isOrigArg()) {
5121 assert(!Res &&
"Call operand has unhandled type");
5126 for (
unsigned i = 0, e =
Ins.size();
i !=
e; ++
i) {
5129 if (Ins[
i].Flags.isByVal()) {
5133 int Size =
Ins[
i].Flags.getByValSize();
5134 unsigned NumRegs = (
Size + 7) / 8;
5146 if (Ins[
i].Flags.isSwiftAsync())
5150 if (
VA.isRegLoc()) {
5152 EVT RegVT =
VA.getLocVT();
5156 RC = &AArch64::GPR32RegClass;
5158 RC = &AArch64::GPR64RegClass;
5160 RC = &AArch64::FPR16RegClass;
5162 RC = &AArch64::FPR32RegClass;
5164 RC = &AArch64::FPR64RegClass;
5166 RC = &AArch64::FPR128RegClass;
5169 RC = &AArch64::PPRRegClass;
5171 RC = &AArch64::ZPRRegClass;
5182 switch (
VA.getLocInfo()) {
5188 assert(
VA.getValVT().isScalableVector() &&
5189 "Only scalable vectors can be passed indirectly");
5205 assert(
VA.isMemLoc() &&
"CCValAssign is neither reg nor mem");
5208 ?
VA.getLocVT().getSizeInBits()
5209 :
VA.getValVT().getSizeInBits()) / 8;
5213 !
Ins[
i].Flags.isInConsecutiveRegs())
5225 switch (
VA.getLocInfo()) {
5233 assert(
VA.getValVT().isScalableVector() &&
5234 "Only scalable vectors can be passed indirectly");
5249 ExtType,
DL,
VA.getLocVT(), Chain,
FIN,
5255 assert(
VA.getValVT().isScalableVector() &&
5256 "Only scalable vectors can be passed indirectly");
5258 uint64_t
PartSize =
VA.getValVT().getStoreSize().getKnownMinSize();
5260 if (Ins[
i].Flags.isInConsecutiveRegs()) {
5261 assert(!Ins[
i].Flags.isInConsecutiveRegsLast());
5262 while (!Ins[
i +
NumParts - 1].Flags.isInConsecutiveRegsLast())
5273 InVals.push_back(ArgValue);
5291 InVals.push_back(ArgValue);
5305 saveVarArgRegisters(CCInfo, DAG,
DL, Chain);
5309 unsigned StackOffset = CCInfo.getNextStackOffset();
5325 if (!CCInfo.isAllocated(AArch64::X8)) {
5336 for (
unsigned I = 0,
E =
Ins.size();
I !=
E; ++
I) {
5337 if (Ins[
I].Flags.isInReg()) {
5354 if (DoesCalleeRestoreStack(CallConv,
TailCallOpt)) {
5378void AArch64TargetLowering::saveVarArgRegisters(
CCState &CCInfo,
5391 AArch64::X3, AArch64::X4, AArch64::X5,
5392 AArch64::X6, AArch64::X7 };
5429 AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
5430 AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
5464SDValue AArch64TargetLowering::LowerCallResult(
5478 for (
unsigned i = 0;
i !=
RVLocs.size(); ++
i) {
5485 "unexpected return calling convention register assignment");
5501 switch (
VA.getLocInfo()) {
5520 InVals.push_back(Val);
5548bool AArch64TargetLowering::isEligibleForTailCallOptimization(
5583 if (
i->hasByValAttr())
5592 if (
i->hasInRegAttr())
5610 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
5621 "Unexpected variadic calling convention");
5624 if (isVarArg && !Outs.empty()) {
5675 A.getValVT().isScalableVector()) &&
5676 "Expected value to be scalable");
5712 if (FI->getIndex() < 0) {
5735AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
5744 bool &IsTailCall = CLI.IsTailCall;
5746 bool IsVarArg = CLI.IsVarArg;
5764 return In.VT.isScalableVector();
5773 IsTailCall = isEligibleForTailCallOptimization(
5774 Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
5786 if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
5788 "site marked musttail");
5798 unsigned NumArgs = Outs.size();
5800 for (
unsigned i = 0;
i != NumArgs; ++
i) {
5801 MVT ArgVT = Outs[
i].VT;
5804 "currently not supported");
5814 assert(!Res &&
"Call operand has unhandled type");
5824 unsigned NumArgs = Outs.size();
5825 for (
unsigned i = 0;
i != NumArgs; ++
i) {
5826 MVT ValVT = Outs[
i].VT;
5829 CLI.getArgs()[Outs[
i].OrigArgIndex].Ty,
5841 assert(!Res &&
"Call operand has unhandled type");
5867 NumBytes =
alignTo(NumBytes, 16);
5884 assert(FPDiff % 16 == 0 &&
"unaligned stack on tail call");
5900 if (IsVarArg && CLI.CB && CLI.CB->isMustTailCall()) {
5910 for (
unsigned i = 0, e = Outs.size();
i !=
e; ++
i) {
5916 switch (
VA.getLocInfo()) {
5951 assert(
VA.getValVT().isScalableVector() &&
5952 "Only scalable vectors can be passed indirectly");
5954 uint64_t StoreSize =
VA.getValVT().getStoreSize().getKnownMinSize();
5957 if (Outs[
i].Flags.isInConsecutiveRegs()) {
5958 assert(!Outs[
i].Flags.isInConsecutiveRegsLast());
5959 while (!Outs[
i +
NumParts - 1].Flags.isInConsecutiveRegsLast())
5979 Chain = DAG.
getStore(Chain,
DL, OutVals[
i], Ptr, MPI);
6000 if (
VA.isRegLoc()) {
6001 if (
i == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
6004 "unexpected calling convention register assignment");
6006 "unexpected use of 'returned'");
6016 [=](
const std::pair<unsigned, SDValue> &
Elt) {
6017 return Elt.first ==
VA.getLocReg();
6025 return ArgReg.Reg ==
VA.getLocReg();
6032 CSInfo.emplace_back(
VA.getLocReg(),
i);
6046 OpSize =
VA.getLocVT().getFixedSizeInBits();
6048 OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
6049 :
VA.getValVT().getSizeInBits();
6052 !Flags.isInConsecutiveRegs()) {
6072 Chain = addTokenForArgument(Chain, DAG, MF.
getFrameInfo(), FI);
6081 if (Outs[
i].Flags.isByVal()) {
6086 Outs[
i].Flags.getNonZeroByValAlign(),
6121 auto GV =
G->getGlobal();
6134 const char *Sym = S->getSymbol();
6138 const char *Sym = S->getSymbol();
6153 std::vector<SDValue> Ops;
6154 Ops.push_back(Chain);
6167 Ops.push_back(DAG.getRegister(
RegToPass.first,
6175 Mask =
TRI->getThisReturnPreservedMask(MF, CallConv);
6178 Mask =
TRI->getCallPreservedMask(MF, CallConv);
6181 Mask =
TRI->getCallPreservedMask(MF, CallConv);
6184 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
6186 if (
TRI->isAnyArgRegReserved(MF))
6187 TRI->emitReservedArgRegCallError(MF);
6189 assert(Mask &&
"Missing call preserved mask for calling convention");
6212 "tail calls cannot be marked with clang.arc.attachedcall");
6233 return LowerCallResult(Chain,
InFlag, CallConv, IsVarArg, Ins,
DL, DAG,
6238bool AArch64TargetLowering::CanLowerReturn(
6269 assert(
VA.isRegLoc() &&
"Can only return in registers!");
6272 switch (
VA.getLocInfo()) {
6302 return Elt.first ==
VA.getLocReg();
6312 for (
auto &RetVal :
RetVals) {
6313 Chain = DAG.
getCopyToReg(Chain,
DL, RetVal.first, RetVal.second, Flag);
6316 DAG.
getRegister(RetVal.first, RetVal.second.getValueType()));
6340 if (AArch64::GPR64RegClass.
contains(*
I))
6342 else if (AArch64::FPR64RegClass.
contains(*
I))
6364 unsigned Flag)
const {
6366 N->getOffset(), Flag);
6371 unsigned Flag)
const {
6377 unsigned Flag)
const {
6379 N->getOffset(), Flag);
6384 unsigned Flag)
const {
6389template <
class NodeTy>
6391 unsigned Flags)
const {
6402template <
class NodeTy>
6404 unsigned Flags)
const {
6418template <
class NodeTy>
6420 unsigned Flags)
const {
6432template <
class NodeTy>
6434 unsigned Flags)
const {
6438 SDValue Sym = getTargetNode(
N, Ty, DAG, Flags);
6450 "unexpected offset in global node");
6503AArch64TargetLowering::LowerDarwinGlobalTLSAddress(
SDValue Op,
6506 "This function expects a Darwin target");
6677AArch64TargetLowering::LowerELFGlobalTLSAddress(
SDValue Op,
6693 "in local exec TLS model");
6764AArch64TargetLowering::LowerWindowsGlobalTLSAddress(
SDValue Op,
6803 Chain =
TLS.getValue(1);
6829 return LowerDarwinGlobalTLSAddress(Op, DAG);
6831 return LowerELFGlobalTLSAddress(Op, DAG);
6833 return LowerWindowsGlobalTLSAddress(Op, DAG);
6866 bool ProduceNonFlagSettingCondBr =
6877 if (!RHS.getNode()) {
6904 if (LHS.getValueType().isInteger()) {
6905 assert((LHS.getValueType() == RHS.getValueType()) &&
6911 if (
RHSC &&
RHSC->getZExtValue() == 0 && ProduceNonFlagSettingCondBr) {
6921 uint64_t
Mask = LHS.getConstantOperandVal(1);
6937 uint64_t
Mask = LHS.getConstantOperandVal(1);
6955 LHS.getOpcode() !=
ISD::AND && ProduceNonFlagSettingCondBr) {
6993 EVT VT =
Op.getValueType();
7000 if (
SrcVT.bitsLT(VT))
7002 else if (
SrcVT.bitsGT(VT))
7067 Attribute::NoImplicitFloat))
7083 EVT VT =
Op.getValueType();
7114 "Unexpected type for custom ctpop lowering");
7121 unsigned EltSize = 8;
7136 EVT VT =
Op.getValueType();
7138 useSVEForFixedLengthVectorVT(VT,
true));
7147 EVT VT =
Op.getValueType();
7150 useSVEForFixedLengthVectorVT(VT,
true))
7197 if (
Op.getValueType().isVector())
7198 return LowerVSETCC(Op, DAG);
7205 Chain =
Op.getOperand(0);
7212 EVT VT =
Op.getValueType();
7223 if (!RHS.getNode()) {
7224 assert(LHS.getValueType() ==
Op.getValueType() &&
7225 "Unexpected setcc expansion!");
7230 if (LHS.getValueType().isInteger()) {
7294 if (!RHS.getNode()) {
7307 if (LHS.getValueType().isInteger()) {
7308 assert((LHS.getValueType() == RHS.getValueType()) &&
7319 LHS.getValueType() ==
TVal.getValueType()) {
7320 EVT VT = LHS.getValueType();
7363 if (TrueVal == ~FalseVal) {
7365 }
else if (FalseVal > std::numeric_limits<int64_t>::min() &&
7366 TrueVal == -FalseVal) {
7385 }
else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
7388 if (TrueVal > FalseVal) {
7443 assert(LHS.getValueType() == RHS.getValueType());
7461 CTVal &&
CTVal->isZero() &&
TVal.getValueType() == LHS.getValueType())
7465 FVal.getValueType() == LHS.getValueType())
7488 EVT Ty =
Op.getValueType();
7489 auto Idx =
Op.getConstantOperandAPInt(2);
7490 if (
Idx.sge(-1) &&
Idx.slt(Ty.getVectorMinNumElements()))
7503 return LowerSELECT_CC(CC, LHS, RHS,
TVal,
FVal,
DL, DAG);
7513 EVT Ty =
Op.getValueType();
7514 if (Ty.isScalableVector()) {
7521 if (useSVEForFixedLengthVectorVT(Ty)) {
7560 return LowerSELECT_CC(CC, LHS, RHS,
TVal,
FVal,
DL, DAG);
7571 return getAddrLarge(JT, DAG);
7573 return getAddrTiny(JT, DAG);
7575 return getAddr(JT, DAG);
7604 return getGOT(CP, DAG);
7606 return getAddrLarge(CP, DAG);
7608 return getAddrTiny(CP, DAG);
7610 return getAddr(CP, DAG);
7619 return getAddrLarge(BA, DAG);
7621 return getAddrTiny(BA, DAG);
7623 return getAddr(BA, DAG);
7648 : FuncInfo->getVarArgsStackIndex(),
7739 return LowerWin64_VASTART(Op, DAG);
7741 return LowerDarwin_VASTART(Op, DAG);
7743 return LowerAAPCS_VASTART(Op, DAG);
7755 : Subtarget->isTargetILP32() ? 20 : 32;
7767 "automatic va_arg instruction only works on Darwin");
7770 EVT VT =
Op.getValueType();
7780 Chain =
VAList.getValue(1);
7785 "currently not supported");
7839 EVT VT =
Op.getValueType();
7865#define GET_REGISTER_MATCHER
7866#include "AArch64GenAsmMatcher.inc"
7873 if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
7875 unsigned DwarfRegNum =
MRI->getDwarfRegNum(Reg,
false);
7889 EVT VT =
Op.getValueType();
7905 EVT VT =
Op.getValueType();
7918 unsigned Reg = MF.
addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
7955 bool OptForSize)
const {
7982 unsigned Limit = (OptForSize ? 1 : (Subtarget->
hasFuseLiterals() ? 5 : 2));
7987 <<
" imm value: "; Imm.
dump(););
7999 if (ST->hasNEON() &&
8011 return DAG.
getNode(Opcode,
SDLoc(Operand), VT, Operand);
8021 EVT VT =
Op.getValueType();
8028AArch64TargetLowering::getSqrtResultForDenormInput(
SDValue Op,
8120const char *AArch64TargetLowering::LowerXConstraint(
EVT ConstraintVT)
const {
8150 if (Constraint ==
"Upa")
8152 if (Constraint ==
"Upl")
8160AArch64TargetLowering::getConstraintType(
StringRef Constraint)
const {
8161 if (Constraint.
size() == 1) {
8162 switch (Constraint[0]) {
8196AArch64TargetLowering::getSingleConstraintMatchWeight(
8199 Value *CallOperandVal =
info.CallOperandVal;
8202 if (!CallOperandVal)
8213 if (type->isFloatingPointTy() || type->isVectorTy())
8227std::pair<unsigned, const TargetRegisterClass *>
8228AArch64TargetLowering::getRegForInlineAsmConstraint(
8230 if (Constraint.
size() == 1) {
8231 switch (Constraint[0]) {
8234 return std::make_pair(0U,
nullptr);
8236 return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass);
8238 return std::make_pair(0U, &AArch64::GPR64commonRegClass);
8239 return std::make_pair(0U, &AArch64::GPR32commonRegClass);
8245 return std::make_pair(0U, &AArch64::ZPRRegClass);
8246 return std::make_pair(0U,
nullptr);
8250 return std::make_pair(0U, &AArch64::FPR16RegClass);
8252 return std::make_pair(0U, &AArch64::FPR32RegClass);
8254 return std::make_pair(0U, &AArch64::FPR64RegClass);
8256 return std::make_pair(0U, &AArch64::FPR128RegClass);
8265 return std::make_pair(0U, &AArch64::ZPR_4bRegClass);
8267 return std::make_pair(0U, &AArch64::FPR128_loRegClass);
8273 return std::make_pair(0U, &AArch64::ZPR_3bRegClass);
8280 return std::make_pair(0U,
nullptr);
8282 return restricted ? std::make_pair(0U, &AArch64::PPR_3bRegClass)
8286 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
8287 return std::make_pair(
unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
8291 std::pair<unsigned, const TargetRegisterClass *> Res;
8297 if ((
Size == 4 ||
Size == 5) && Constraint[0] ==
'{' &&
8298 tolower(Constraint[1]) ==
'v' && Constraint[
Size - 1] ==
'}') {
8301 if (!
Failed && RegNo >= 0 && RegNo <= 31) {
8306 Res.first = AArch64::FPR64RegClass.getRegister(RegNo);
8307 Res.second = &AArch64::FPR64RegClass;
8309 Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
8310 Res.second = &AArch64::FPR128RegClass;
8316 if (Res.second && !Subtarget->
hasFPARMv8() &&
8317 !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
8318 !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
8319 return std::make_pair(0U,
nullptr);
8327 if (Subtarget->
hasLS64() && Ty->isIntegerTy(512))
8335void AArch64TargetLowering::LowerAsmOperandForConstraint(
8336 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
8341 if (Constraint.length() != 1)
8366 GA->getValueType(0));
8387 uint64_t CVal =
C->getZExtValue();
8400 uint64_t
NVal = -
C->getSExtValue();
8402 CVal =
C->getSExtValue();
8433 if ((CVal & 0xFFFF) == CVal)
8435 if ((CVal & 0xFFFF0000ULL) == CVal)
8447 if ((CVal & 0xFFFFULL) == CVal)
8449 if ((CVal & 0xFFFF0000ULL) == CVal)
8451 if ((CVal & 0xFFFF00000000ULL) == CVal)
8453 if ((CVal & 0xFFFF000000000000ULL) == CVal)
8455 uint64_t
NCVal = ~CVal;
8462 if ((
NCVal & 0xFFFF000000000000ULL) ==
NCVal)
8476 Ops.push_back(Result);
8503 EVT EltType = V.getValueType().getVectorElementType();
8504 return EltType.getSizeInBits() / 8;
8524 LLVM_DEBUG(
dbgs() <<
"AArch64TargetLowering::ReconstructShuffle\n");
8526 EVT VT = Op.getValueType();
8528 "Scalable vectors cannot be used with ISD::BUILD_VECTOR");
8547 : Vec(Vec),
MinElt(std::numeric_limits<unsigned>::max()),
MaxElt(0),
8563 dbgs() <<
"Reshuffle failed: "
8564 "a shuffle can only come from building a vector from "
8565 "various elements of other vectors, provided their "
8566 "indices are constant\n");
8573 if (Source == Sources.end())
8578 Source->MinElt = std::min(Source->MinElt,
EltNo);
8579 Source->MaxElt = std::max(Source->MaxElt,
EltNo);
8582 if (Sources.size() > 2) {
8584 dbgs() <<
"Reshuffle failed: currently only do something sane when at "
8585 "most two source vectors are involved\n");
8592 for (
auto &Source : Sources) {
8593 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
8607 for (
auto &Src : Sources) {
8608 EVT SrcVT = Src.ShuffleVec.getValueType();
8626 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8632 dbgs() <<
"Reshuffle failed: result vector too small to extract\n");
8638 dbgs() <<
"Reshuffle failed: span too large for a VEXT to cope\n");
8663 if (!
SrcVT.is64BitVector()) {
8665 dbgs() <<
"Reshuffle failed: don't know how to lower AArch64ISD::EXT "
8666 "for SVE vectors.");
8673 Src.WindowBase = -Src.MinElt;
8680 for (
auto &Src : Sources) {
8681 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8688 Src.WindowBase *= Src.WindowScale;
8701 if (Entry.isUndef())
8704 auto Src =
find(Sources, Entry.getOperand(0));
8710 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8727 LLVM_DEBUG(
dbgs() <<
"Reshuffle failed: illegal shuffle mask\n");
8732 for (
unsigned i = 0;
i < Sources.size(); ++
i)
8740 dbgs() <<
"Reshuffle, creating node: "; V.dump(););
8783 "Only possible block sizes for wide DUP are: 16, 32, 64");
8823 "Shuffle with all-undefs must have been caught by previous cases, "
8865 [&](
int Elt) {return Elt != ExpectedElt++ && Elt != -1;});
8896 "Only possible block sizes for REV are: 16, 32, 64");
8928 if ((M[
i] >= 0 && (
unsigned)M[
i] !=
Idx) ||
8973 if ((M[
i] >= 0 && (
unsigned)M[
i] !=
Idx) ||
8988 for (
unsigned j = 0;
j != 2; ++
j) {
8990 for (
unsigned i = 0;
i != Half; ++
i) {
8991 int MIdx = M[
i +
j * Half];
9078 EVT VT = Op.getValueType();
9087 bool SplitV0 =
V0.getValueSizeInBits() == 128;
9097 if (
V1.getValueSizeInBits() == 128) {
9109 unsigned OpNum = (
PFEntry >> 26) & 0x0F;
9132 if (
LHSID == (1 * 9 + 2) * 9 + 3)
9134 assert(
LHSID == ((4 * 9 + 5) * 9 + 6) * 9 + 7 &&
"Illegal OP_COPY!");
9213 SDValue V2 = Op.getOperand(1);
9216 EVT EltVT = Op.getValueType().getVectorElementType();
9220 for (
int Val : ShuffleMask) {
9221 for (
unsigned Byte = 0; Byte <
BytesPerElt; ++Byte) {
9229 if (Op.getValueSizeInBits() == 128) {
9238 if (V2.getNode()->isUndef()) {
9323 Lane += V.getConstantOperandVal(1);
9324 V = V.getOperand(0);
9341 EVT VT =
Op.getValueType();
9345 if (useSVEForFixedLengthVectorVT(VT))
9346 return LowerFixedLengthVECTOR_SHUFFLEToSVE(Op, DAG);
9357 assert(
V1.getValueType() == VT &&
"Unexpected VECTOR_SHUFFLE type!");
9359 "Unexpected VECTOR_SHUFFLE mask size!");
9361 if (
SVN->isSplat()) {
9362 int Lane =
SVN->getSplatIndex();
9377 unsigned Opcode =
getDUPLANEOp(
V1.getValueType().getVectorElementType());
9382 for (
unsigned LaneSize : {64U, 32U, 16U}) {
9432 return DAG.
getNode(Opc, dl,
V1.getValueType(),
V1, V2);
9436 return DAG.
getNode(Opc, dl,
V1.getValueType(),
V1, V2);
9440 return DAG.
getNode(Opc, dl,
V1.getValueType(),
V1, V2);
9490 for (
unsigned i = 0;
i != 4; ++
i) {
9491 if (ShuffleMask[
i] < 0)
9501 unsigned Cost = (
PFEntry >> 30);
9513 EVT VT =
Op.getValueType();
9517 if (useSVEForFixedLengthVectorVT(VT))
9518 return LowerToScalableOp(Op, DAG);
9522 switch (
ElemVT.getSimpleVT().SimpleTy) {
9527 if (ConstVal->isOne())
9528 return getPTrue(DAG, dl, VT, AArch64SVEPredPattern::all);
9566 EVT VT =
Op.getValueType();
9580 if (
CIdx && (
CIdx->getZExtValue() <= 3)) {
9611 EVT VT =
BVN->getValueType(0);
9612 APInt SplatBits, SplatUndef;
9613 unsigned SplatBitSize;
9615 if (
BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
HasAnyUndefs)) {
9633 const APInt &Bits) {
9634 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9635 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9636 EVT VT = Op.getValueType();
9655 const SDValue *LHS =
nullptr) {
9656 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9657 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9658 EVT VT = Op.getValueType();
9703 const SDValue *LHS =
nullptr) {
9704 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9705 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9706 EVT VT = Op.getValueType();
9743 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9744 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9745 EVT VT = Op.getValueType();
9773 const APInt &Bits) {
9774 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9775 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9776 EVT VT = Op.getValueType();
9794 const APInt &Bits) {
9795 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9796 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9797 EVT VT = Op.getValueType();
9827 uint64_t &ConstVal) {
9834 EVT VT =
Bvec->getValueType(0);
9839 ConstVal =
FirstElt->getZExtValue();
9844 unsigned Opcode =
N->getOpcode();
9850 if (IID < Intrinsic::num_intrinsics)
9864 EVT VT =
N->getValueType(0);
9919 uint64_t
C2 =
C2node->getZExtValue();
9947 if (useSVEForFixedLengthVectorVT(
Op.getValueType()))
9948 return LowerToScalableOp(Op, DAG);
9954 EVT VT =
Op.getValueType();
9961 LHS =
Op.getOperand(1);
9995 EVT VT = Op.getValueType();
10002 for (
SDValue Lane : Op->ops()) {
10011 }
else if (Lane.getNode()->isUndef()) {
10015 "Unexpected BUILD_VECTOR operand type");
10017 Ops.push_back(Lane);
10023 EVT VT = Op.getValueType();
10065 EVT VT =
Op.getValueType();
10075 if (
BVN->isConstant())
10079 Const->getAPIntValue().zextOrTrunc(BitSize).getZExtValue());
10080 if (Val.isNullValue() || Val.isAllOnesValue())
10127 if (!ConstantValue.
getNode())
10129 else if (ConstantValue != V)
10133 if (!
Value.getNode())
10135 else if (V !=
Value) {
10141 if (!
Value.getNode()) {
10143 dbgs() <<
"LowerBUILD_VECTOR: value undefined, creating undef node\n");
10151 LLVM_DEBUG(
dbgs() <<
"LowerBUILD_VECTOR: only low element used, creating 1 "
10152 "SCALAR_TO_VECTOR node\n");
10164 const SDNode *
N = V.getNode();
10186 if (Val == 2 *
i) {
10190 if (Val - 1 == 2 *
i) {
10222 Value.getValueType() != VT) {
10224 dbgs() <<
"LowerBUILD_VECTOR: use DUP for non-constant splats\n");
10232 if (
Value.getValueSizeInBits() == 64) {
10234 dbgs() <<
"LowerBUILD_VECTOR: DUPLANE works on 128-bit vectors, "
10247 EltTy ==
MVT::f64) &&
"Unsupported floating-point vector type");
10249 dbgs() <<
"LowerBUILD_VECTOR: float constant splats, creating int "
10250 "BITCASTS, and try again\n");
10256 LLVM_DEBUG(
dbgs() <<
"LowerBUILD_VECTOR: trying to lower new vector: ";
10258 Val = LowerBUILD_VECTOR(Val, DAG);
10302 dbgs() <<
"LowerBUILD_VECTOR: all elements are constant, use default "
10335 dbgs() <<
"LowerBUILD_VECTOR: alternatives failed, creating sequence "
10336 "of INSERT_VECTOR_ELT\n");
10353 LLVM_DEBUG(
dbgs() <<
"Creating node for op0, it is not undefined:\n");
10358 <<
"Creating nodes for the other vector elements:\n";);
10370 dbgs() <<
"LowerBUILD_VECTOR: use default expansion, failed to find "
10371 "better alternative\n");
10377 if (useSVEForFixedLengthVectorVT(
Op.getValueType()))
10378 return LowerFixedLengthConcatVectorsToSVE(Op, DAG);
10380 assert(
Op.getValueType().isScalableVector() &&
10382 "Expected legal scalable vector type!");
10384 if (
isTypeLegal(
Op.getOperand(0).getValueType()) &&
Op.getNumOperands() == 2)
10390SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(
SDValue Op,
10394 if (useSVEForFixedLengthVectorVT(
Op.getValueType()))
10395 return LowerFixedLengthInsertVectorElt(Op, DAG);
10398 EVT VT =
Op.getOperand(0).getValueType();
10407 VectorVT.getScalarType().getSizeInBits() < 32
10412 ExtendedValue,
Op.getOperand(2));
10438 Op.getOperand(1),
Op.getOperand(2));
10444AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(
SDValue Op,
10447 EVT VT =
Op.getOperand(0).getValueType();
10462 if (useSVEForFixedLengthVectorVT(VT))
10463 return LowerFixedLengthExtractVectorElt(Op, DAG);
10496SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(
SDValue Op,
10498 assert(
Op.getValueType().isFixedLengthVector() &&
10499 "Only cases that extract a fixed length vector are supported!");
10501 EVT InVT =
Op.getOperand(0).getValueType();
10503 unsigned Size =
Op.getValueSizeInBits();
10505 if (
InVT.isScalableVector()) {
10514 if (
Idx == 0 &&
InVT.getSizeInBits() <= 128)
10519 if (
Size == 64 &&
Idx *
InVT.getScalarSizeInBits() == 64 &&
10520 InVT.getSizeInBits() == 128)
10528 assert(
Op.getValueType().isScalableVector() &&
10529 "Only expect to lower inserts into scalable vectors!");
10531 EVT InVT =
Op.getOperand(1).getValueType();
10534 if (
InVT.isScalableVector()) {
10536 EVT VT =
Op.getValueType();
10555 }
else if (
Idx ==
InVT.getVectorMinNumElements()) {
10571 EVT VT =
Op.getValueType();
10573 if (useSVEForFixedLengthVectorVT(VT,
true))
10574 return LowerFixedLengthVectorIntDivideToSVE(Op, DAG);
10582 return LowerToPredicatedOp(Op, DAG,
PredOpcode);
10608 if (useSVEForFixedLengthVectorVT(VT))
10614 for (
unsigned i = 0;
i != 4; ++
i) {
10625 unsigned Cost = (
PFEntry >> 30);
10654 Op = Op.getOperand(0);
10656 APInt SplatBits, SplatUndef;
10657 unsigned SplatBitSize;
10659 if (!
BVN || !
BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
10672 assert(VT.
isVector() &&
"vector shift count is not a vector type");
10683 assert(VT.
isVector() &&
"vector shift count is not a vector type");
10692 EVT VT =
Op.getValueType();
10697 EVT OpVT =
Op.getOperand(0).getValueType();
10707 if (useSVEForFixedLengthVectorVT(
Op.getOperand(0).getValueType()))
10708 return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
10713SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(
SDValue Op,
10715 EVT VT =
Op.getValueType();
10719 if (!
Op.getOperand(1).getValueType().isVector())
10723 switch (
Op.getOpcode()) {
10731 if (
isVShiftLImm(
Op.getOperand(1), VT,
false, Cnt) && Cnt < EltSize)
10737 Op.getOperand(0),
Op.getOperand(1));
10743 return LowerToPredicatedOp(Op, DAG, Opc);
10747 if (
isVShiftRImm(
Op.getOperand(1), VT,
false, Cnt) && Cnt < EltSize) {
10750 return DAG.
getNode(Opc,
DL, VT,
Op.getOperand(0),
10757 unsigned Opc = (
Op.getOpcode() ==
ISD::SRA) ? Intrinsic::aarch64_neon_sshl
10777 "function only supposed to emit natural comparisons");
10785 if (
SrcVT.getVectorElementType().isFloatingPoint()) {
10869 if (
Op.getValueType().isScalableVector())
10872 if (useSVEForFixedLengthVectorVT(
Op.getOperand(0).getValueType()))
10873 return LowerFixedLengthVectorSetccToSVE(Op, DAG);
10878 EVT CmpVT = LHS.getValueType().changeVectorElementTypeToInteger();
10881 if (LHS.getValueType().getVectorElementType().isInteger()) {
10882 assert(LHS.getValueType() == RHS.getValueType());
10895 if (LHS.getValueType().getVectorNumElements() == 4) {
10906 LHS.getValueType().getVectorElementType() !=
MVT::f128);
10917 if (!
Cmp.getNode())
10957 if (
SrcVT.isScalableVector() ||
10961 return LowerPredReductionToSVE(Op, DAG);
10963 switch (
Op.getOpcode()) {
10993 switch (
Op.getOpcode()) {
11029 MVT VT =
Op.getSimpleValueType();
11034 Op.getOperand(0),
Op.getOperand(1), RHS,
11035 AN->getMemOperand());
11046 MVT VT =
Op.getSimpleValueType();
11051 Op.getOperand(0),
Op.getOperand(1), RHS,
11052 AN->getMemOperand());
11055SDValue AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(
11084AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(
SDValue Op,
11087 "Only Windows alloca probing supported");
11095 EVT VT =
Node->getValueType(0);
11098 "no-stack-arg-probe")) {
11106 SDValue Ops[2] = {SP, Chain};
11112 Chain = LowerWindowsDYNAMIC_STACKALLOC(Op, Chain,
Size, DAG);
11125 SDValue Ops[2] = {SP, Chain};
11131 EVT VT =
Op.getValueType();
11141template <
unsigned NumVecs>
11160 Info.align.reset();
11171 unsigned Intrinsic)
const {
11172 auto &
DL =
I.getModule()->getDataLayout();
11173 switch (Intrinsic) {
11174 case Intrinsic::aarch64_sve_st2:
11176 case Intrinsic::aarch64_sve_st3:
11178 case Intrinsic::aarch64_sve_st4:
11180 case Intrinsic::aarch64_neon_ld2:
11181 case Intrinsic::aarch64_neon_ld3:
11182 case Intrinsic::aarch64_neon_ld4:
11183 case Intrinsic::aarch64_neon_ld1x2:
11184 case Intrinsic::aarch64_neon_ld1x3:
11185 case Intrinsic::aarch64_neon_ld1x4:
11186 case Intrinsic::aarch64_neon_ld2lane:
11187 case Intrinsic::aarch64_neon_ld3lane:
11188 case Intrinsic::aarch64_neon_ld4lane:
11189 case Intrinsic::aarch64_neon_ld2r:
11190 case Intrinsic::aarch64_neon_ld3r:
11191 case Intrinsic::aarch64_neon_ld4r: {
11194 uint64_t
NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
11196 Info.ptrVal =
I.getArgOperand(
I.getNumArgOperands() - 1);
11198 Info.align.reset();
11203 case Intrinsic::aarch64_neon_st2:
11204 case Intrinsic::aarch64_neon_st3:
11205 case Intrinsic::aarch64_neon_st4:
11206 case Intrinsic::aarch64_neon_st1x2:
11207 case Intrinsic::aarch64_neon_st1x3:
11208 case Intrinsic::aarch64_neon_st1x4:
11209 case Intrinsic::aarch64_neon_st2lane:
11210 case Intrinsic::aarch64_neon_st3lane:
11211 case Intrinsic::aarch64_neon_st4lane: {
11217 if (!
ArgTy->isVectorTy())
11222 Info.ptrVal =
I.getArgOperand(
I.getNumArgOperands() - 1);
11224 Info.align.reset();
11229 case Intrinsic::aarch64_ldaxr:
11230 case Intrinsic::aarch64_ldxr: {
11234 Info.ptrVal =
I.getArgOperand(0);
11240 case Intrinsic::aarch64_stlxr:
11241 case Intrinsic::aarch64_stxr: {
11245 Info.ptrVal =
I.getArgOperand(1);
11251 case Intrinsic::aarch64_ldaxp:
11252 case Intrinsic::aarch64_ldxp:
11255 Info.ptrVal =
I.getArgOperand(0);
11257 Info.align =
Align(16);
11260 case Intrinsic::aarch64_stlxp:
11261 case Intrinsic::aarch64_stxp:
11264 Info.ptrVal =
I.getArgOperand(2);
11266 Info.align =
Align(16);
11269 case Intrinsic::aarch64_sve_ldnt1: {
11273 Info.ptrVal =
I.getArgOperand(1);
11277 if (Intrinsic == Intrinsic::aarch64_sve_ldnt1)
11281 case Intrinsic::aarch64_sve_stnt1: {
11284 Info.memVT =
MVT::getVT(
I.getOperand(0)->getType());
11285 Info.ptrVal =
I.getArgOperand(2);
11289 if (Intrinsic == Intrinsic::aarch64_sve_stnt1)
11318 Base.getOperand(1).hasOneUse() &&
11333 if (!
Ty1->isIntegerTy() || !
Ty2->isIntegerTy())
11335 uint64_t
NumBits1 =
Ty1->getPrimitiveSizeInBits().getFixedSize();
11336 uint64_t
NumBits2 =
Ty2->getPrimitiveSizeInBits().getFixedSize();
11340 if (
VT1.isVector() ||
VT2.isVector() || !
VT1.isInteger() || !
VT2.isInteger())
11351 if (
I->getOpcode() != Instruction::FMul)
11354 if (!
I->hasOneUse())
11360 !(
User->getOpcode() == Instruction::FSub ||
11361 User->getOpcode() == Instruction::FAdd))
11378 if (!
Ty1->isIntegerTy() || !
Ty2->isIntegerTy())
11380 unsigned NumBits1 =
Ty1->getPrimitiveSizeInBits();
11381 unsigned NumBits2 =
Ty2->getPrimitiveSizeInBits();
11385 if (
VT1.isVector() ||
VT2.isVector() || !
VT1.isInteger() || !
VT2.isInteger())
11402 return (
VT1.isSimple() && !
VT1.isVector() &&
VT1.isInteger() &&
11403 VT2.isSimple() && !
VT2.isVector() &&
VT2.isInteger() &&
11404 VT1.getSizeInBits() <= 32);
11407bool AArch64TargetLowering::isExtFreeImpl(
const Instruction *Ext)
const {
11412 if (Ext->getType()->isVectorTy())
11415 for (
const Use &U : Ext->
uses()) {
11424 case Instruction::Shl:
11430 auto &
DL = Ext->getModule()->getDataLayout();
11431 std::advance(
GTI, U.getOperandNo()-1);
11437 uint64_t ShiftAmt =
11441 if (ShiftAmt == 0 || ShiftAmt > 4)
11445 case Instruction::Trunc:
11448 if (Instr->
getType() == Ext->getOperand(0)->getType())
11466 auto *HalfTy =
HalfV->getType();
11467 return FullTy->getPrimitiveSizeInBits().getFixedSize() ==
11468 2 * HalfTy->getPrimitiveSizeInBits().getFixedSize();
11474 return FullVT->getNumElements() == 2 *
HalfVT->getNumElements();
11506 return Ext->getType()->getScalarSizeInBits() ==
11507 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
11540 if (!
I->getType()->isVectorTy())
11544 switch (II->getIntrinsicID()) {
11545 case Intrinsic::aarch64_neon_umull:
11548 Ops.push_back(&II->getOperandUse(0));
11549 Ops.push_back(&II->getOperandUse(1));
11552 case Intrinsic::aarch64_neon_pmull64:
11554 II->getArgOperand(1)))
11556 Ops.push_back(&II->getArgOperandUse(0));
11557 Ops.push_back(&II->getArgOperandUse(1));
11565 switch (
I->getOpcode()) {
11566 case Instruction::Sub:
11567 case Instruction::Add: {
11576 Ops.push_back(&
Ext1->getOperandUse(0));
11577 Ops.push_back(&
Ext2->getOperandUse(0));
11580 Ops.push_back(&
I->getOperandUse(0));
11581 Ops.push_back(&
I->getOperandUse(1));
11585 case Instruction::Mul: {
11587 for (
auto &Op :
I->operands()) {
11589 if (
any_of(Ops, [&](
Use *U) {
return U->get() == Op; }))
11612 if (Opcode != Instruction::SExt && Opcode != Instruction::ZExt)
11616 Ops.push_back(&Op);
11635 unsigned NumBits =
LoadedType.getSizeInBits();
11636 return NumBits == 32 || NumBits == 64;
11644 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
11658 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
11689 "Invalid interleave factor");
11690 assert(!Shuffles.empty() &&
"Empty shufflevector input");
11691 assert(Shuffles.size() == Indices.size() &&
11692 "Unmatched number of shufflevectors and indices");
11710 Type *EltTy =
FVTy->getElementType();
11720 if (NumLoads > 1) {
11724 FVTy->getNumElements() / NumLoads);
11729 BaseAddr = Builder.CreateBitCast(
11737 Intrinsic::aarch64_neon_ld3,
11738 Intrinsic::aarch64_neon_ld4};
11752 BaseAddr = Builder.CreateConstGEP1_32(
FVTy->getElementType(), BaseAddr,
11753 FVTy->getNumElements() * Factor);
11756 LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy),
"ldN");
11759 for (
unsigned i = 0;
i < Shuffles.size();
i++) {
11761 unsigned Index = Indices[
i];
11767 SubVec = Builder.CreateIntToPtr(
11769 FVTy->getNumElements()));
11782 SVI->replaceAllUsesWith(
WideVec);
11816 unsigned Factor)
const {
11818 "Invalid interleave factor");
11821 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
11823 unsigned LaneLen = VecTy->getNumElements() / Factor;
11824 Type *EltTy = VecTy->getElementType();
11844 Type *IntTy =
DL.getIntPtrType(EltTy);
11850 Op0 = Builder.CreatePtrToInt(Op0,
IntVecTy);
11851 Op1 = Builder.CreatePtrToInt(Op1,
IntVecTy);
11857 Value *BaseAddr =
SI->getPointerOperand();
11859 if (NumStores > 1) {
11868 BaseAddr = Builder.CreateBitCast(
11870 SubVecTy->getElementType()->getPointerTo(
SI->getPointerAddressSpace()));
11875 Type *PtrTy =
SubVecTy->getPointerTo(
SI->getPointerAddressSpace());
11878 Intrinsic::aarch64_neon_st3,
11879 Intrinsic::aarch64_neon_st4};
11883 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
11888 for (
unsigned i = 0;
i < Factor;
i++) {
11890 if (Mask[
IdxI] >= 0) {
11891 Ops.push_back(Builder.CreateShuffleVector(
11897 if (Mask[
IdxJ * Factor +
IdxI] >= 0) {
11907 Ops.push_back(Builder.CreateShuffleVector(
11914 if (StoreCount > 0)
11915 BaseAddr = Builder.CreateConstGEP1_32(
SubVecTy->getElementType(),
11918 Ops.push_back(Builder.CreateBitCast(BaseAddr, PtrTy));
11919 Builder.CreateCall(
StNFunc, Ops);
11944SDValue AArch64TargetLowering::LowerSVEStructLoad(
unsigned Intrinsic,
11950 unsigned N, Opcode;
11951 static std::map<unsigned, std::pair<unsigned, unsigned>>
IntrinsicMap = {
11958 "invalid tuple vector type!");
11971 for (
unsigned I = 0;
I <
N; ++
I)
12040 if (
Immed == std::numeric_limits<int64_t>::min()) {
12042 <<
": avoid UB for INT64_MIN\n");
12048 ((
Immed & 0xfff) == 0 &&
Immed >> 24 == 0));
12050 <<
" legal add imm: " << (
IsLegal ?
"yes" :
"no") <<
"\n");
12090 uint64_t NumBytes = 0;
12091 if (Ty->isSized()) {
12092 uint64_t NumBits =
DL.getTypeSizeInBits(Ty);
12093 NumBytes = NumBits / 8;
12106 unsigned shift =
Log2_64(NumBytes);
12107 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
12116 return AM.
Scale == 1 || (AM.
Scale > 0 && (uint64_t)AM.
Scale == NumBytes);
12162 switch (Ty->getScalarType()->getTypeID()) {
12182 AArch64::X16, AArch64::X17, AArch64::LR, 0
12190 N =
N->getOperand(0).getNode();
12191 EVT VT =
N->getValueType(0);
12196 uint64_t
TruncMask =
N->getConstantOperandVal(1);
12198 N->getOperand(0).getOpcode() ==
ISD::SRL &&
12207 assert(Ty->isIntegerTy());
12209 unsigned BitSize = Ty->getPrimitiveSizeInBits();
12217 if ((int64_t)Val < 0)
12220 Val &= (1LL << 32) - 1;
12223 unsigned Shift = (63 - LZ) / 16;
12229 unsigned Index)
const {
12233 return (Index == 0 || Index ==
ResVT.getVectorNumElements());
12242 EVT VT =
N->getValueType(0);
12257 if (!ShiftAmt || ShiftAmt->getZExtValue() !=
ShiftEltTy.getSizeInBits() - 1)
12292 if (ABS->getOperand(0)->getOpcode() !=
ISD::SUB ||
12293 ABS->getOperand(0)->getValueType(0) !=
MVT::v16i32)
12296 SDValue SUB = ABS->getOperand(0);
12297 unsigned Opcode0 = SUB->getOperand(0).getOpcode();
12298 unsigned Opcode1 = SUB->getOperand(1).getOpcode();
12300 if (SUB->getOperand(0)->getValueType(0) !=
MVT::v16i32 ||
12301 SUB->getOperand(1)->getValueType(0) !=
MVT::v16i32)
12305 bool IsZExt =
false;
12358 if (!ST->hasDotProd())
12372 if (
A.getOpcode() !=
B.getOpcode() ||
12373 A.getOperand(0).getValueType() !=
B.getOperand(0).getValueType())
12380 EVT Op0VT =
A.getOperand(0).getValueType();
12390 B =
B.getOperand(0);
12397 A.getOperand(0),
B);
12404 if (
DCI.isBeforeLegalizeOps())
12411AArch64TargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
12419 EVT VT =
N->getValueType(0);
12421 !(Divisor.
isPowerOf2() || (-Divisor).isPowerOf2()))
12457 case Intrinsic::aarch64_sve_cntb:
12458 case Intrinsic::aarch64_sve_cnth:
12459 case Intrinsic::aarch64_sve_cntw:
12460 case Intrinsic::aarch64_sve_cntd:
12479 switch (
Extend.getOpcode()) {
12482 return Extend.getOperand(0).getValueType();
12489 return TypeNode->
getVT();
12541 unsigned ExtendOpcode =
Extend.getOpcode();
12594 if (!
Mul->getValueType(0).isVector())
12606 Op0 ? Op0 :
Mul->getOperand(0),
12617 if (
DCI.isBeforeLegalizeOps())
12658 if (
N->hasOneUse() && (
N->use_begin()->getOpcode() ==
ISD::ADD ||
12659 N->use_begin()->getOpcode() ==
ISD::SUB))
12666 unsigned ShiftAmt, AddSubOpc;
12681 }
else if (
CVPlus1.isPowerOf2()) {
12682 ShiftAmt =
CVPlus1.logBase2();
12704 EVT VT =
N->getValueType(0);
12712 "NegateResult and TrailingZeroes cannot both be true for now.");
12736 EVT VT =
N->getValueType(0);
12738 N->getOperand(0)->getOperand(0)->getOpcode() !=
ISD::SETCC ||
12739 VT.
getSizeInBits() !=
N->getOperand(0)->getValueType(0).getSizeInBits())
12749 if (!
BV->isConstant())
12761 N->getOperand(0)->getOperand(0),
MaskConst);
12776 EVT VT =
N->getValueType(0);
12781 if (VT.
getSizeInBits() !=
N->getOperand(0).getValueSizeInBits())
12793 LN0->getPointerInfo(),
LN0->getAlignment(),
12794 LN0->getMemOperand()->getFlags());
12816 if (!
N->getValueType(0).isSimple())
12820 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12828 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
12834 uint32_t IntBits = IntTy.getSizeInBits();
12835 if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12844 int32_t Bits = IntBits == 64 ? 64 : 32;
12845 int32_t
C =
BV->getConstantFPSplatPow2ToLog2Int(&
UndefElements, Bits + 1);
12846 if (
C == -1 ||
C == 0 ||
C > Bits)
12850 unsigned NumLanes = Op.getValueType().getVectorNumElements();
12866 "Illegal vector type after legalization");
12870 unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs
12871 : Intrinsic::aarch64_neon_vcvtfp2fxu;
12892 unsigned Opc = Op->getOpcode();
12893 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12894 !Op.getOperand(0).getValueType().isSimple() ||
12902 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
12903 int32_t IntBits = IntTy.getSizeInBits();
12904 if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12923 unsigned NumLanes = Op.getValueType().getVectorNumElements();
12945 unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp
12946 : Intrinsic::aarch64_neon_vcvtfxu2fp;
12966 ShiftAmount =
N->getConstantOperandVal(1);
12980 EVT VT =
N->getValueType(0);
13018 EVT VT =
N->getValueType(0);
13041 for (
int i = 1;
i >= 0; --
i) {
13042 for (
int j = 1;
j >= 0; --
j) {
13079 uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1);
13080 for (
int i = 1;
i >= 0; --
i)
13081 for (
int j = 1;
j >= 0; --
j) {
13092 CN0->getZExtValue() != (BitMask & ~
CN1->getZExtValue())) {
13110 EVT VT =
N->getValueType(0);
13125 if (!
MemVT.getVectorElementType().isSimple())
13129 switch (
MemVT.getVectorElementType().getSimpleVT().SimpleTy) {
13146 return Op0->getAPIntValue().getLimitedValue() ==
MaskForTy;
13153 if (
DCI.isBeforeLegalizeOps())
13170 uint64_t
ExtVal =
C->getZExtValue();
13174 EVT EltTy =
UnpkOp->getValueType(0).getVectorElementType();
13186 UnpkOp->getValueType(0),
13244 EVT VT =
N->getValueType(0);
13291 EVT VT =
N->getValueType(0);
13304 uint64_t ShiftAmt =
C->getZExtValue();
13305 if (VT ==
MVT::i32 && ShiftAmt == 16 &&
13308 if (VT ==
MVT::i64 && ShiftAmt == 32 &&
13330 EVT VT =
N->getValueType(0);
13343 if (ShiftAmount != 1)
13414 EVT VT =
N->getValueType(0);
13440 if (Shuffle && Shuffle->
getMaskElt(0) == 1 &&
13457 EVT VT =
N->getValueType(0);
13478 if (
N00VT ==
N10.getValueType() &&
13483 for (
size_t i = 0;
i < Mask.size(); ++
i)
13495 if (
DCI.isBeforeLegalizeOps())
13509 if (
N->getNumOperands() == 2 &&
N0Opc ==
N1Opc &&
13533 uint64_t
N00Index =
N00.getConstantOperandVal(1);
13534 uint64_t
N01Index =
N01.getConstantOperandVal(1);
13535 uint64_t
N10Index =
N10.getConstantOperandVal(1);
13536 uint64_t
N11Index =
N11.getConstantOperandVal(1);
13566 MVT RHSTy = RHS.getValueType().getSimpleVT();
13568 if (!
RHSTy.isVector())
13572 dbgs() <<
"aarch64-lower: concat_vectors bitcast simplification\n");
13575 RHSTy.getVectorNumElements() * 2);
13587 if (
DCI.isBeforeLegalizeOps())
13614 "unexpected vector size on extract_vector_elt!");
13645 switch (
N.getOpcode()) {
13681 N =
N.getOperand(0);
13684 if (
N.getOperand(0).getValueType().isScalableVector())
13687 N.getOperand(0).getValueType().getVectorNumElements() / 2;
13770 isSetCC(Op->getOperand(0), Info));
13780 assert(Op && Op->getOpcode() ==
ISD::ADD &&
"Unexpected operation!");
13781 SDValue LHS = Op->getOperand(0);
13782 SDValue RHS = Op->getOperand(1);
13801 ?
InfoAndKind.Info.AArch64.Cmp->getOperand(0).getValueType()
13802 :
InfoAndKind.Info.Generic.Opnd0->getValueType();
13820 EVT VT = Op->getValueType(0);
13827 EVT VT =
N->getValueType(0);
13849 OpVT1.getVectorElementType() != VT)
13864 EVT VT =
N->getValueType(0);
13899 if (
DCI.isBeforeLegalizeOps())
13902 MVT VT =
N->getSimpleValueType(0);
13914 LHS.getOpcode() != RHS.getOpcode())
13917 unsigned ExtType = LHS.getOpcode();
13923 if (!RHS.getNode())
13929 if (!LHS.getNode())
13960 if (
DCI.isBeforeLegalizeOps())
13965 assert(LHS.getValueType().is64BitVector() &&
13966 RHS.getValueType().is64BitVector() &&
13967 "unexpected shape for long operation");
13974 if (!RHS.getNode())
13978 if (!LHS.getNode())
13983 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N),
N->getValueType(0), LHS, RHS);
13986 N->getOperand(0), LHS, RHS);
13991 unsigned ElemBits = ElemTy.getSizeInBits();
13993 int64_t ShiftAmount;
13996 unsigned SplatBitSize;
13998 if (!
BVN->isConstantSplat(
SplatValue, SplatUndef, SplatBitSize,
14005 ShiftAmount =
CVN->getSExtValue();
14014 case Intrinsic::aarch64_neon_sqshl:
14018 case Intrinsic::aarch64_neon_uqshl:
14022 case Intrinsic::aarch64_neon_srshl:
14026 case Intrinsic::aarch64_neon_urshl:
14030 case Intrinsic::aarch64_neon_sqshlu:
14034 case Intrinsic::aarch64_neon_sshl:
14035 case Intrinsic::aarch64_neon_ushl:
14046 return DAG.
getNode(Opcode, dl,
N->getValueType(0),
N->getOperand(1),
14050 return DAG.
getNode(Opcode, dl,
N->getValueType(0),
N->getOperand(1),
14066 if (!
CMask ||
CMask->getZExtValue() != Mask)
14070 N->getOperand(0),
N->getOperand(1),
AndN.getOperand(0));
14078 N->getOperand(1).getSimpleValueType(),
14101 SDValue Scalar =
N->getOperand(3);
14102 EVT ScalarTy = Scalar.getValueType();
14116 EVT VT =
N->getValueType(0);
14142 if (
DCI.isBeforeLegalize())
14149 EVT VT =
N->getValueType(0);
14150 EVT CmpVT =
N->getOperand(2).getValueType();
14161 case Intrinsic::aarch64_sve_cmpeq_wide:
14162 case Intrinsic::aarch64_sve_cmpne_wide:
14163 case Intrinsic::aarch64_sve_cmpge_wide:
14164 case Intrinsic::aarch64_sve_cmpgt_wide:
14165 case Intrinsic::aarch64_sve_cmplt_wide:
14166 case Intrinsic::aarch64_sve_cmple_wide: {
14168 int64_t ImmVal =
CN->getSExtValue();
14169 if (ImmVal >= -16 && ImmVal <= 15)
14177 case Intrinsic::aarch64_sve_cmphs_wide:
14178 case Intrinsic::aarch64_sve_cmphi_wide:
14179 case Intrinsic::aarch64_sve_cmplo_wide:
14180 case Intrinsic::aarch64_sve_cmpls_wide: {
14182 uint64_t ImmVal =
CN->getZExtValue();
14208 assert(Op.getValueType().isScalableVector() &&
14210 "Expected legal scalable vector type!");
14287 unsigned NumElts =
N.getValueType().getVectorMinNumElements();
14291 N =
N.getOperand(0);
14294 if (
N.getValueType().getVectorMinNumElements() <
NumElts)
14302 N.getConstantOperandVal(0) == AArch64SVEPredPattern::all)
14303 return N.getValueType().getVectorMinNumElements() >=
NumElts;
14315 assert(
N->getNumOperands() == 4 &&
"Expected 3 operand intrinsic!");
14321 return DAG.
getNode(Opc,
SDLoc(
N),
N->getValueType(0),
N->getOperand(2),
14340 case Intrinsic::aarch64_neon_vcvtfxs2fp:
14341 case Intrinsic::aarch64_neon_vcvtfxu2fp:
14343 case Intrinsic::aarch64_neon_saddv:
14345 case Intrinsic::aarch64_neon_uaddv:
14347 case Intrinsic::aarch64_neon_sminv:
14349 case Intrinsic::aarch64_neon_uminv:
14351 case Intrinsic::aarch64_neon_smaxv:
14353 case Intrinsic::aarch64_neon_umaxv:
14355 case Intrinsic::aarch64_neon_fmax:
14357 N->getOperand(1),
N->getOperand(2));
14358 case Intrinsic::aarch64_neon_fmin:
14360 N->getOperand(1),
N->getOperand(2));
14361 case Intrinsic::aarch64_neon_fmaxnm:
14363 N->getOperand(1),
N->getOperand(2));
14364 case Intrinsic::aarch64_neon_fminnm:
14366 N->getOperand(1),
N->getOperand(2));
14367 case Intrinsic::aarch64_neon_smull:
14368 case Intrinsic::aarch64_neon_umull:
14369 case Intrinsic::aarch64_neon_pmull:
14370 case Intrinsic::aarch64_neon_sqdmull:
14372 case Intrinsic::aarch64_neon_sqshl:
14373 case Intrinsic::aarch64_neon_uqshl:
14374 case Intrinsic::aarch64_neon_sqshlu:
14375 case Intrinsic::aarch64_neon_srshl:
14376 case Intrinsic::aarch64_neon_urshl:
14377 case Intrinsic::aarch64_neon_sshl:
14378 case Intrinsic::aarch64_neon_ushl:
14380 case Intrinsic::aarch64_crc32b:
14381 case Intrinsic::aarch64_crc32cb:
14383 case Intrinsic::aarch64_crc32h:
14384 case Intrinsic::aarch64_crc32ch:
14386 case Intrinsic::aarch64_sve_saddv:
14388 if (
N->getOperand(2)->getValueType(0).getVectorElementType() ==
MVT::i64)
14392 case Intrinsic::aarch64_sve_uaddv:
14394 case Intrinsic::aarch64_sve_smaxv:
14396 case Intrinsic::aarch64_sve_umaxv:
14398 case Intrinsic::aarch64_sve_sminv:
14400 case Intrinsic::aarch64_sve_uminv:
14402 case Intrinsic::aarch64_sve_orv:
14404 case Intrinsic::aarch64_sve_eorv:
14406 case Intrinsic::aarch64_sve_andv:
14408 case Intrinsic::aarch64_sve_index:
14410 case Intrinsic::aarch64_sve_dup:
14412 case Intrinsic::aarch64_sve_dup_x:
14415 case Intrinsic::aarch64_sve_ext:
14417 case Intrinsic::aarch64_sve_mul:
14419 case Intrinsic::aarch64_sve_smulh:
14421 case Intrinsic::aarch64_sve_umulh:
14423 case Intrinsic::aarch64_sve_smin:
14425 case Intrinsic::aarch64_sve_umin:
14427 case Intrinsic::aarch64_sve_smax:
14429 case Intrinsic::aarch64_sve_umax:
14431 case Intrinsic::aarch64_sve_lsl:
14433 case Intrinsic::aarch64_sve_lsr:
14435 case Intrinsic::aarch64_sve_asr:
14437 case Intrinsic::aarch64_sve_fadd:
14439 case Intrinsic::aarch64_sve_fsub:
14441 case Intrinsic::aarch64_sve_fmul:
14443 case Intrinsic::aarch64_sve_add:
14445 case Intrinsic::aarch64_sve_sub:
14447 case Intrinsic::aarch64_sve_and:
14449 case Intrinsic::aarch64_sve_bic:
14451 case Intrinsic::aarch64_sve_eor:
14453 case Intrinsic::aarch64_sve_orr:
14455 case Intrinsic::aarch64_sve_sqadd:
14457 case Intrinsic::aarch64_sve_sqsub:
14459 case Intrinsic::aarch64_sve_uqadd:
14461 case Intrinsic::aarch64_sve_uqsub:
14463 case Intrinsic::aarch64_sve_sqadd_x:
14465 N->getOperand(1),
N->getOperand(2));
14466 case Intrinsic::aarch64_sve_sqsub_x:
14468 N->getOperand(1),
N->getOperand(2));
14469 case Intrinsic::aarch64_sve_uqadd_x:
14471 N->getOperand(1),
N->getOperand(2));
14472 case Intrinsic::aarch64_sve_uqsub_x:
14474 N->getOperand(1),
N->getOperand(2));
14475 case Intrinsic::aarch64_sve_cmphs:
14476 if (!
N->getOperand(2).getValueType().isFloatingPoint())
14478 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
14481 case Intrinsic::aarch64_sve_cmphi:
14482 if (!
N->getOperand(2).getValueType().isFloatingPoint())
14484 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
14487 case Intrinsic::aarch64_sve_fcmpge:
14488 case Intrinsic::aarch64_sve_cmpge:
14490 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
14493 case Intrinsic::aarch64_sve_fcmpgt:
14494 case Intrinsic::aarch64_sve_cmpgt:
14496 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
14499 case Intrinsic::aarch64_sve_fcmpeq:
14500 case Intrinsic::aarch64_sve_cmpeq:
14502 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
14505 case Intrinsic::aarch64_sve_fcmpne:
14506 case Intrinsic::aarch64_sve_cmpne:
14508 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
14511 case Intrinsic::aarch64_sve_fcmpuo:
14513 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
14516 case Intrinsic::aarch64_sve_fadda:
14518 case Intrinsic::aarch64_sve_faddv:
14520 case Intrinsic::aarch64_sve_fmaxnmv:
14522 case Intrinsic::aarch64_sve_fmaxv:
14524 case Intrinsic::aarch64_sve_fminnmv:
14526 case Intrinsic::aarch64_sve_fminv:
14528 case Intrinsic::aarch64_sve_sel:
14530 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
14531 case Intrinsic::aarch64_sve_cmpeq_wide:
14533 case Intrinsic::aarch64_sve_cmpne_wide:
14535 case Intrinsic::aarch64_sve_cmpge_wide:
14537 case Intrinsic::aarch64_sve_cmpgt_wide:
14539 case Intrinsic::aarch64_sve_cmplt_wide:
14541 case Intrinsic::aarch64_sve_cmple_wide:
14543 case Intrinsic::aarch64_sve_cmphs_wide:
14545 case Intrinsic::aarch64_sve_cmphi_wide:
14547 case Intrinsic::aarch64_sve_cmplo_wide:
14549 case Intrinsic::aarch64_sve_cmpls_wide:
14551 case Intrinsic::aarch64_sve_ptest_any:
14554 case Intrinsic::aarch64_sve_ptest_first:
14557 case Intrinsic::aarch64_sve_ptest_last:
14572 (
N->getOperand(0).getOpcode() ==
ISD::ABDU ||
14573 N->getOperand(0).getOpcode() ==
ISD::ABDS)) {
14587 assert(!
St.isTruncatingStore() &&
"cannot split truncating vector store");
14597 uint64_t BaseOffset = 0;
14605 if (BasePtr->getOpcode() ==
ISD::ADD &&
14608 BasePtr = BasePtr->getOperand(0);
14619 St.getMemOperand()->getFlags());
14628 assert(
ContentTy.isSimple() &&
"No SVE containers for extended types");
14630 switch (
ContentTy.getSimpleVT().SimpleTy) {
14657 EVT VT =
N->getValueType(0);
14683 EVT VT =
N->getValueType(0);
14684 EVT PtrTy =
N->getOperand(3).getValueType();
14710template <
unsigned Opcode>
14714 "Unsupported opcode.");
14716 EVT VT =
N->getValueType(0);
14746 if (
DataVT.isFloatingPoint())
14750 if (
Data.getValueType().isFloatingPoint())
14770 EVT PtrTy =
N->getOperand(4).getValueType();
14776 if (
DataVT.isFloatingPoint())
14824 if (!
StVal.hasOneUse())
14829 if (
St.isTruncatingStore())
14835 int64_t
Offset =
St.getBasePtr()->getConstantOperandVal(1);
14884 if (
St.isTruncatingStore())
14996 if (
N->getOperand(2).isUndef())
14997 return N->getOperand(1);
15028 unsigned Opc =
N->getOpcode();
15034 "Invalid opcode.");
15093 unsigned OpScalarSize = Op.getScalarValueSizeInBits();
15095 unsigned ShiftImm =
N->getConstantOperandVal(1);
15112 if (
DCI.isBeforeLegalizeOps())
15116 EVT VT =
N->getValueType(0);
15146 if (UI.getUse().getResNo() == 1)
15156 Addr.getNode()->use_end(); UI != UE; ++UI) {
15159 || UI.getUse().getResNo() !=
Addr.getResNo())
15176 Visited.insert(
Addr.getNode());
15177 Worklist.push_back(
User);
15178 Worklist.push_back(LD);
15179 Worklist.push_back(
Vector.getNode());
15185 Ops.push_back(LD->getOperand(0));
15188 Ops.push_back(Lane);
15190 Ops.push_back(
Addr);
15191 Ops.push_back(Inc);
15222 !
DCI.isBeforeLegalizeOps());
15225 DCI.CommitTargetLoweringOpt(
TLO);
15233 "Expected STORE dag node in input!");
15236 if (!Store->isTruncatingStore() || Store->isIndexed())
15238 SDValue Ext = Store->getValue();
15244 if (Store->getMemoryVT() !=
Orig->getValueType(0))
15247 Store->getBasePtr(), Store->getPointerInfo(),
15248 Store->getAlign());
15276 if (
DCI.isBeforeLegalize() ||
DCI.isCalledByLegalizer())
15279 unsigned AddrOpIdx =
N->getNumOperands() - 1;
15284 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
15287 UI.getUse().getResNo() !=
Addr.getResNo())
15294 Visited.insert(
Addr.getNode());
15295 Worklist.push_back(
N);
15296 Worklist.push_back(
User);
15302 bool IsStore =
false;
15317 NumVecs = 2; IsStore =
true;
break;
15319 NumVecs = 3; IsStore =
true;
break;
15321 NumVecs = 4; IsStore =
true;
break;
15329 NumVecs = 2; IsStore =
true;
break;
15331 NumVecs = 3; IsStore =
true;
break;
15333 NumVecs = 4; IsStore =
true;
break;
15356 VecTy =
N->getOperand(2).getValueType();
15358 VecTy =
N->getValueType(0);
15372 Ops.push_back(
N->getOperand(0));
15376 Ops.push_back(
N->getOperand(
i));
15377 Ops.push_back(
Addr);
15378 Ops.push_back(Inc);
15393 MemInt->getMemOperand());
15414 switch(V.getNode()->getOpcode()) {
15421 ExtType =
LoadNode->getExtensionType();
15447 1LL << (width - 1);
15615 else if (
CNV == 65535)
15687 unsigned CmpOpc = Cmp.getOpcode();
15693 if (!Cmp->hasNUsesOfValue(0, 0) || !Cmp->hasNUsesOfValue(1, 1))
15696 SDValue LHS = Cmp.getOperand(0);
15697 SDValue RHS = Cmp.getOperand(1);
15699 assert(LHS.getValueType() == RHS.getValueType() &&
15700 "Expected the value type to be the same for both operands!");
15722 DCI.CombineTo(
N, BR,
false);
15732 if (
N->getOperand(0) ==
N->getOperand(1))
15733 return N->getOperand(0);
15748 LHS->hasOneUse()) {
15754 auto NewCond = getInvertedCondCode(
OldCond);
15769 "Unexpected opcode!");
15780 LHS->getOperand(0)->getValueType(0) ==
N->getValueType(0) &&
15782 LHS->getOperand(0)->getOperand(0) == Pred)
15783 return LHS->getOperand(0);
15795 if (!Op->hasOneUse())
15805 Bit < Op->getValueType(0).getSizeInBits()) {
15811 Bit < Op->getOperand(0).getValueSizeInBits()) {
15815 if (Op->getNumOperands() != 2)
15822 switch (Op->getOpcode()) {
15828 if ((
C->getZExtValue() >> Bit) & 1)
15834 if (
C->getZExtValue() <= Bit &&
15835 (Bit -
C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
15836 Bit = Bit -
C->getZExtValue();
15843 Bit = Bit +
C->getZExtValue();
15844 if (Bit >= Op->getValueType(0).getSizeInBits())
15845 Bit = Op->getValueType(0).getSizeInBits() - 1;
15850 if ((Bit +
C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
15851 Bit = Bit +
C->getZExtValue();
15858 if ((
C->getZExtValue() >> Bit) & 1)
15876 unsigned NewOpc =
N->getOpcode();
15912 if (CmpLHS.
getValueType() ==
N->getOperand(1).getValueType() &&
15942 if (
ResVT.getSizeInBits() !=
CmpVT.getSizeInBits())
15967 if (
ResVT.isScalableVector())
15974 "Scalar-SETCC feeding SELECT has unexpected result type!");
15996 if (
CCVT.getSizeInBits() !=
ResVT.getSizeInBits())
16016 ResVT.changeVectorElementTypeToInteger(), Mask);
16023 if (
N->getValueType(0) ==
N->getOperand(0).getValueType())
16024 return N->getOperand(0);
16040 uint64_t MinOffset = -1ull;
16049 MinOffset = std::min(MinOffset,
C->getZExtValue());
16051 uint64_t
Offset = MinOffset +
GN->getOffset();
16056 if (
Offset <= uint64_t(
GN->getOffset()))
16067 if (
Offset >= (1 << 21))
16072 if (!T->isSized() ||
16087 "This method is only for scalable vectors of offsets");
16135 "Scatter stores are only possible for SVE vectors");
16145 if (
SrcElVT.isFloatingPoint())
16181 SrcVT.getScalarSizeInBits() / 8)) {
16192 if (!TLI.isTypeLegal(
Base.getValueType()))
16202 if (!TLI.isTypeLegal(
Offset.getValueType()))
16212 if (
SrcVT.isFloatingPoint())
16230 return DAG.
getNode(Opcode,
DL, VTs, Ops);
16238 "Gather loads are only possible for SVE vectors");
16258 RetVT.getScalarSizeInBits());
16268 Offset.getValueType().isVector())
16280 RetVT.getScalarSizeInBits() / 8)) {
16295 if (!TLI.isTypeLegal(
Base.getValueType()))
16312 if (
RetVT.isFloatingPoint())
16328 if (
RetVT.isFloatingPoint())
16361 "Sign extending from an invalid type");
16371 if (
DCI.isBeforeLegalizeOps())
16503 Ops[1] = DAG.
getConstant(Intrinsic::aarch64_sve_prfb_gather_uxtw_index,
DL,
16512 switch (Op.getOpcode()) {
16560 if (
N->getValueType(0) !=
ExtractVec.getValueType())
16579 EVT Ty =
N->getValueType(0);
16580 if (Ty.isInteger())
16585 if (
ExtIntTy.getVectorElementType().getScalarSizeInBits() <
16586 IntTy.getVectorElementType().getScalarSizeInBits())
16603 switch (
N->getOpcode()) {
16698 case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
16700 case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
16702 case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
16704 case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
16706 case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
16707 case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
16708 case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
16709 case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
16710 case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
16711 case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
16712 case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
16713 case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
16715 case Intrinsic::aarch64_neon_ld2:
16716 case Intrinsic::aarch64_neon_ld3:
16717 case Intrinsic::aarch64_neon_ld4:
16718 case Intrinsic::aarch64_neon_ld1x2:
16719 case Intrinsic::aarch64_neon_ld1x3:
16720 case Intrinsic::aarch64_neon_ld1x4:
16721 case Intrinsic::aarch64_neon_ld2lane:
16722 case Intrinsic::aarch64_neon_ld3lane:
16723 case Intrinsic::aarch64_neon_ld4lane:
16724 case Intrinsic::aarch64_neon_ld2r:
16725 case Intrinsic::aarch64_neon_ld3r:
16726 case Intrinsic::aarch64_neon_ld4r:
16727 case Intrinsic::aarch64_neon_st2:
16728 case Intrinsic::aarch64_neon_st3:
16729 case Intrinsic::aarch64_neon_st4:
16730 case Intrinsic::aarch64_neon_st1x2:
16731 case Intrinsic::aarch64_neon_st1x3:
16732 case Intrinsic::aarch64_neon_st1x4:
16733 case Intrinsic::aarch64_neon_st2lane:
16734 case Intrinsic::aarch64_neon_st3lane:
16735 case Intrinsic::aarch64_neon_st4lane:
16737 case Intrinsic::aarch64_sve_ldnt1:
16739 case Intrinsic::aarch64_sve_ld1rq:
16741 case Intrinsic::aarch64_sve_ld1ro:
16743 case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
16745 case Intrinsic::aarch64_sve_ldnt1_gather:
16747 case Intrinsic::aarch64_sve_ldnt1_gather_index:
16750 case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
16752 case Intrinsic::aarch64_sve_ld1:
16754 case Intrinsic::aarch64_sve_ldnf1:
16756 case Intrinsic::aarch64_sve_ldff1:
16758 case Intrinsic::aarch64_sve_st1:
16760 case Intrinsic::aarch64_sve_stnt1:
16762 case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
16764 case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
16766 case Intrinsic::aarch64_sve_stnt1_scatter:
16768 case Intrinsic::aarch64_sve_stnt1_scatter_index:
16770 case Intrinsic::aarch64_sve_ld1_gather:
16772 case Intrinsic::aarch64_sve_ld1_gather_index:
16775 case Intrinsic::aarch64_sve_ld1_gather_sxtw:
16778 case Intrinsic::aarch64_sve_ld1_gather_uxtw:
16781 case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
16785 case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
16789 case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
16791 case Intrinsic::aarch64_sve_ldff1_gather:
16793 case Intrinsic::aarch64_sve_ldff1_gather_index:
16796 case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
16800 case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
16804 case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
16808 case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
16812 case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
16815 case Intrinsic::aarch64_sve_st1_scatter:
16817 case Intrinsic::aarch64_sve_st1_scatter_index:
16819 case Intrinsic::aarch64_sve_st1_scatter_sxtw:
16822 case Intrinsic::aarch64_sve_st1_scatter_uxtw:
16825 case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
16829 case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
16833 case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
16835 case Intrinsic::aarch64_sve_tuple_get: {
16843 uint64_t
NumLanes =
ResVT.getVectorElementCount().getKnownMinValue();
16849 case Intrinsic::aarch64_sve_tuple_set: {
16871 Opnds.push_back(Vec);
16882 case Intrinsic::aarch64_sve_tuple_create2:
16883 case Intrinsic::aarch64_sve_tuple_create3:
16884 case Intrinsic::aarch64_sve_tuple_create4: {
16889 for (
unsigned I = 2;
I <
N->getNumOperands(); ++
I)
16890 Opnds.push_back(
N->getOperand(
I));
16896 (
N->getNumOperands() - 2));
16900 case Intrinsic::aarch64_sve_ld2:
16901 case Intrinsic::aarch64_sve_ld3:
16902 case Intrinsic::aarch64_sve_ld4: {
16906 SDValue BasePtr =
N->getOperand(3);
16908 unsigned IntrinsicID =
16911 LowerSVEStructLoad(IntrinsicID,
LoadOps,
N->getValueType(0), DAG,
DL);
16914 case Intrinsic::aarch64_rndr:
16915 case Intrinsic::aarch64_rndrrs: {
16916 unsigned IntrinsicID =
16919 (IntrinsicID == Intrinsic::aarch64_rndr ? AArch64SysReg::RNDR
16920 : AArch64SysReg::RNDRRS);
16946bool AArch64TargetLowering::isUsedByReturnOnly(
SDNode *
N,
16948 if (
N->getNumValues() != 1)
16950 if (!
N->hasNUsesOfValue(1, 0))
16954 SDNode *Copy = *
N->use_begin();
16958 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() ==
16961 TCChain = Copy->getOperand(0);
16983bool AArch64TargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
16995 Base =
Op->getOperand(0);
16999 int64_t
RHSC = RHS->getSExtValue();
17018 VT =
LD->getMemoryVT();
17019 Ptr =
LD->getBasePtr();
17021 VT =
ST->getMemoryVT();
17022 Ptr =
ST->getBasePtr();
17033bool AArch64TargetLowering::getPostIndexedAddressParts(
17039 VT =
LD->getMemoryVT();
17040 Ptr =
LD->getBasePtr();
17042 VT =
ST->getMemoryVT();
17043 Ptr =
ST->getBasePtr();
17058void AArch64TargetLowering::ReplaceBITCASTResults(
17062 EVT VT =
N->getValueType(0);
17067 "Expected fp->int bitcast!");
17105 return std::make_pair(Lo, Hi);
17108void AArch64TargetLowering::ReplaceExtractSubVectorResults(
17114 if (!
InVT.isScalableVector() || !
InVT.isInteger())
17118 EVT VT =
N->getValueType(0);
17124 if (
InVT.getVectorElementCount() != (
ResEC * 2))
17131 unsigned Index =
CIndex->getZExtValue();
17132 if ((Index != 0) && (Index !=
ResEC.getKnownMinValue()))
17144 SDLoc dl(V.getNode());
17165 "AtomicCmpSwap on types less than 128 should be legal");
17179 switch (
MemOp->getMergedOrdering()) {
17181 Opcode = AArch64::CASPX;
17184 Opcode = AArch64::CASPAX;
17187 Opcode = AArch64::CASPLX;
17191 Opcode = AArch64::CASPALX;
17215 switch (
MemOp->getMergedOrdering()) {
17217 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
17220 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
17223 Opcode = AArch64::CMP_SWAP_128_RELEASE;
17227 Opcode = AArch64::CMP_SWAP_128;
17236 New.first, New.second,
N->getOperand(0)};
17247void AArch64TargetLowering::ReplaceNodeResults(
17249 switch (
N->getOpcode()) {
17253 ReplaceBITCASTResults(
N,
Results, DAG);
17287 assert(
N->getValueType(0) ==
MVT::i128 &&
"unexpected illegal conversion");
17295 "unexpected load's value type");
17305 DAG.
getVTList({MVT::i64, MVT::i64, MVT::Other}),
17306 {LoadNode->getChain(), LoadNode->getBasePtr()},
LoadNode->getMemoryVT(),
17315 ReplaceExtractSubVectorResults(
N,
Results, DAG);
17322 EVT VT =
N->getValueType(0);
17324 "custom lowering for unexpected type");
17331 case Intrinsic::aarch64_sve_clasta_n: {
17335 N->getOperand(1), Op2,
N->getOperand(3));
17339 case Intrinsic::aarch64_sve_clastb_n: {
17343 N->getOperand(1), Op2,
N->getOperand(3));
17347 case Intrinsic::aarch64_sve_lasta: {
17350 N->getOperand(1),
N->getOperand(2));
17354 case Intrinsic::aarch64_sve_lastb: {
17357 N->getOperand(1),
N->getOperand(2));
17372unsigned AArch64TargetLowering::combineRepeatedFPDivisors()
const {
17393 unsigned Size =
SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
17394 return Size == 128;
17418 if (Subtarget->
hasLSE())
17473 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
17479 if (ValueTy->getPrimitiveSizeInBits() == 128) {
17481 IsAcquire ? Intrinsic::aarch64_ldaxp : Intrinsic::aarch64_ldxp;
17487 Value *Lo = Builder.CreateExtractValue(
LoHi, 0,
"lo");
17488 Value *Hi = Builder.CreateExtractValue(
LoHi, 1,
"hi");
17489 Lo = Builder.CreateZExt(Lo, ValueTy,
"lo64");
17490 Hi = Builder.CreateZExt(Hi, ValueTy,
"hi64");
17491 return Builder.CreateOr(
17497 IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
17504 return Builder.CreateBitCast(Trunc, ValueTy);
17509 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
17516 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
17524 IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp;
17528 Value *Lo = Builder.CreateTrunc(Val, Int64Ty,
"lo");
17529 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty,
"hi");
17531 return Builder.CreateCall(
Stxr, {Lo, Hi,
Addr});
17535 IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
17541 Val = Builder.CreateBitCast(Val,
IntValTy);
17543 return Builder.CreateCall(
Stxr,
17544 {Builder.CreateZExtOrBitCast(
17545 Val,
Stxr->getFunctionType()->getParamType(0)),
17552 if (!Ty->isArrayTy()) {
17554 return TySize.isScalable() &&
TySize.getKnownMinSize() > 128;
17563bool AArch64TargetLowering::shouldNormalizeToSelectSequence(
LLVMContext &,
17597 M.getOrInsertGlobal(
"__security_cookie",
17606 F->addAttribute(1, Attribute::AttrKind::InReg);
17616 return M.getGlobalVariable(
"__security_cookie");
17653 return Mask->getValue().isPowerOf2();
17696 if (AArch64::GPR64RegClass.
contains(*
I))
17697 RC = &AArch64::GPR64RegClass;
17698 else if (AArch64::FPR64RegClass.
contains(*
I))
17699 RC = &AArch64::FPR64RegClass;
17709 assert(Entry->getParent()->getFunction().hasFnAttribute(
17710 Attribute::NoUnwind) &&
17711 "Function should be nounwind in insertCopiesSplitCSR!");
17712 Entry->addLiveIn(*
I);
17717 for (
auto *Exit : Exits)
17719 TII->get(TargetOpcode::COPY), *
I)
17753void AArch64TargetLowering::finalizeLowering(
MachineFunction &MF)
const {
17763bool AArch64TargetLowering::shouldLocalize(
17765 switch (
MI.getOpcode()) {
17766 case TargetOpcode::G_GLOBAL_VALUE: {
17777 case AArch64::ADRP:
17778 case AArch64::G_ADD_LOW:
17806 "Expected legal fixed length vector!");
17832 "Expected legal fixed length vector!");
17839 PgPattern = AArch64SVEPredPattern::vl1;
17842 PgPattern = AArch64SVEPredPattern::vl2;
17845 PgPattern = AArch64SVEPredPattern::vl4;
17848 PgPattern = AArch64SVEPredPattern::vl8;
17851 PgPattern = AArch64SVEPredPattern::vl16;
17854 PgPattern = AArch64SVEPredPattern::vl32;
17857 PgPattern = AArch64SVEPredPattern::vl64;
17860 PgPattern = AArch64SVEPredPattern::vl128;
17863 PgPattern = AArch64SVEPredPattern::vl256;
17899 "Expected legal scalable vector!");
17914 "Expected to convert into a scalable vector!");
17915 assert(V.getValueType().isFixedLengthVector() &&
17916 "Expected a fixed length vector operand!");
17925 "Expected to convert into a fixed length vector!");
17926 assert(V.getValueType().isScalableVector() &&
17927 "Expected a scalable vector operand!");
17934SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
17939 EVT VT =
Op.getValueType();
17945 Load->getMemoryVT(),
Load->getMemOperand(),
Load->getAddressingMode(),
17946 Load->getExtensionType());
17956 EVT InVT = Mask.getValueType();
17969SDValue AArch64TargetLowering::LowerFixedLengthVectorMLoadToSVE(
17977 EVT VT =
Op.getValueType();
17985 if (
Load->getPassThru()->isUndef()) {
18000 Load->getAddressingMode(),
Load->getExtensionType());
18014SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
18019 EVT VT =
Store->getValue().getValueType();
18026 Store->getMemOperand(),
Store->getAddressingMode(),
18027 Store->isTruncatingStore());
18030SDValue AArch64TargetLowering::LowerFixedLengthVectorMStoreToSVE(
18034 if (
Store->isTruncatingStore())
18038 EVT VT =
Store->getValue().getValueType();
18046 Mask,
Store->getMemoryVT(),
Store->getMemOperand(),
18047 Store->getAddressingMode(),
Store->isTruncatingStore());
18050SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
18053 EVT VT =
Op.getValueType();
18061 return LowerToPredicatedOp(Op, DAG,
PredOpcode,
true);
18110SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
18112 EVT VT =
Op.getValueType();
18146SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
18148 EVT VT =
Op.getValueType();
18182SDValue AArch64TargetLowering::LowerFixedLengthExtractVectorElt(
18184 EVT VT =
Op.getValueType();
18185 EVT InVT =
Op.getOperand(0).getValueType();
18186 assert(
InVT.isFixedLengthVector() &&
"Expected fixed length vector type!");
18195SDValue AArch64TargetLowering::LowerFixedLengthInsertVectorElt(
18197 EVT VT =
Op.getValueType();
18201 EVT InVT =
Op.getOperand(0).getValueType();
18206 Op.getOperand(1),
Op.getOperand(2));
18218 EVT VT =
Op.getValueType();
18227 for (
const SDValue &V :
Op->op_values()) {
18241 "Only fixed length vectors are supported!");
18255 for (
const SDValue &V :
Op->op_values()) {
18256 assert((!V.getValueType().isVector() ||
18257 V.getValueType().isScalableVector()) &&
18258 "Only scalable vectors are supported!");
18273 EVT VT =
Op.getValueType();
18274 assert(useSVEForFixedLengthVectorVT(VT) &&
18275 "Only expected to lower fixed length vector operation!");
18280 for (
const SDValue &V :
Op->op_values()) {
18284 if (!V.getValueType().isVector()) {
18290 assert(useSVEForFixedLengthVectorVT(V.getValueType()) &&
18291 "Only fixed length vectors are supported!");
18308 if (
SrcVT.isFixedLengthVector()) {
18334 if (!
OpVT.isScalableVector() ||
OpVT.getVectorElementType() !=
MVT::i1)
18360SDValue AArch64TargetLowering::LowerReductionToSVE(
unsigned Opcode,
18367 if (useSVEForFixedLengthVectorVT(
SrcVT,
true)) {
18374 SrcVT.getVectorElementType();
18392AArch64TargetLowering::LowerFixedLengthVectorSelectToSVE(
SDValue Op,
18394 EVT VT =
Op.getValueType();
18397 EVT InVT =
Op.getOperand(1).getValueType();
18416SDValue AArch64TargetLowering::LowerFixedLengthVectorSetccToSVE(
18419 EVT InVT =
Op.getOperand(0).getValueType();
18422 assert(useSVEForFixedLengthVectorVT(
InVT) &&
18423 "Only expected to lower fixed length vector operation!");
18424 assert(
Op.getValueType() ==
InVT.changeTypeToInteger() &&
18425 "Expected integer result of the same bit length as the inputs!");
18433 {Pg, Op1, Op2,
Op.getOperand(2)});
18441AArch64TargetLowering::LowerFixedLengthBitcastToSVE(
SDValue Op,
18444 auto SrcOp =
Op.getOperand(0);
18445 EVT VT =
Op.getValueType();
18455SDValue AArch64TargetLowering::LowerFixedLengthConcatVectorsToSVE(
18458 unsigned NumOperands =
Op->getNumOperands();
18461 "Unexpected number of operands in CONCAT_VECTORS");
18465 EVT VT =
Op.getValueType();
18468 if (NumOperands > 2) {
18471 for (
unsigned I = 0;
I < NumOperands;
I += 2)
18473 Op->getOperand(
I),
Op->getOperand(
I + 1)));
18490AArch64TargetLowering::LowerFixedLengthFPExtendToSVE(
SDValue Op,
18492 EVT VT =
Op.getValueType();
18501 SrcVT.getVectorElementType());
18507 Val = getSVESafeBitCast(
ExtendVT, Val, DAG);
18515AArch64TargetLowering::LowerFixedLengthFPRoundToSVE(
SDValue Op,
18517 EVT VT =
Op.getValueType();
18531 Val = getSVESafeBitCast(
ContainerSrcVT.changeTypeToInteger(), Val, DAG);
18539AArch64TargetLowering::LowerFixedLengthIntToFPToSVE(
SDValue Op,
18541 EVT VT =
Op.getValueType();
18559 VT.changeTypeToInteger(), Val);
18562 Val = getSVESafeBitCast(
ContainerDstVT.changeTypeToInteger(), Val, DAG);
18584AArch64TargetLowering::LowerFixedLengthFPToIntToSVE(
SDValue Op,
18586 EVT VT =
Op.getValueType();
18609 Val = getSVESafeBitCast(
CvtVT, Val, DAG);
18627SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
18629 EVT VT =
Op.getValueType();
18633 auto ShuffleMask =
SVN->getMask();
18672 "Only expect to cast between legal scalable vector types!");
18675 "Cannot cast between data and predicate scalable vector types!");
18700 return ::isAllActivePredicate(
N);
18704 return ::getPromotedVTForPredicate(VT);
18707bool AArch64TargetLowering::SimplifyDemandedBitsForTargetNode(
18710 unsigned Depth)
const {
18712 unsigned Opc = Op.getOpcode();
18732 unsigned ScalarSize = Op.getScalarValueSizeInBits();
18743 return TLO.CombineTo(Op,
ShiftR->getOperand(0));
18751bool AArch64TargetLowering::isConstantUnsignedBitfieldExtactLegal(
unsigned const MachineRegisterInfo * MRI
static unsigned MatchRegisterName(StringRef Name)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isConcatMask(ArrayRef< int > Mask, EVT VT, bool SplitLHS)
static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG)
static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, const SDLoc &dl, SelectionDAG &DAG)
static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A))...
static void changeVectorFPCCToAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector...
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isSingletonEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue performCONDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex)
static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue NormalizeBuildVector(SDValue Op, SelectionDAG &DAG)
static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St)
Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
static SDValue GenerateTBL(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static std::pair< SDValue, SDValue > splitInt128(SDValue N, SelectionDAG &DAG)
static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo)
Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.
static bool isLegalArithImmed(uint64_t C)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG)
unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend)
static SDValue convertFixedMaskToScalableVector(SDValue Mask, SelectionDAG &DAG)
static bool areExtractShuffleVectors(Value *Op1, Value *Op2)
Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector ...
unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend)
static bool isREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
static SDValue performSRLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isZerosVector(const SDNode *N)
isZerosVector - Check whether SDNode N is a zero-filled vector.
static SDValue performGLD1Combine(SDNode *N, SelectionDAG &DAG)
static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
Fold a floating-point divide by power of two into fixed-point to floating-point conversion.
static SDValue convertMergedOpToPredOp(SDNode *N, unsigned Opc, SelectionDAG &DAG, bool UnpredOp=false)
static SDValue getScaledOffsetForBitWidth(SelectionDAG &DAG, SDValue Offset, SDLoc DL, unsigned BitWidth)
static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG)
static bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType)
static SDValue performSVEAndCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performBRCONDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static MVT getSVEContainerType(EVT ContentTy)
static bool isMergePassthruOpcode(unsigned Opc)
static EVT calculatePreExtendType(SDValue Extend, SelectionDAG &DAG)
Calculates what the pre-extend type is, based on the extension operation node provided by Extend.
static SDValue performNEONPostLDSTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.
static void ReplaceCMP_SWAP_128Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static SDValue performSelectCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instruct...
static cl::opt< bool > EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true))
static bool isUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool isValidImmForSVEVecImmAddrMode(unsigned OffsetInBytes, unsigned ScalarSizeInBytes)
Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scat...
static bool isUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v,...
static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static unsigned getDUPLANEOp(EVT EltType)
static void changeFPCCToAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
static bool isTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget, const TargetMachine &TM)
static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST, EVT VT, EVT MemVT, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static bool isLanes1toNKnownZero(SDValue Op)
static bool setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL, AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI)
Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.
static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG)
static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N, SelectionDAG &DAG)
static SDValue performNVCASTCombine(SDNode *N)
Get rid of unnecessary NVCASTs (that don't change the type).
static EVT getPackedSVEVectorVT(EVT VT)
static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG)
static SDValue performSpliceCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performCSELCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static void ReplaceReductionResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, unsigned InterOp, unsigned AcrossOp)
static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount, bool &FromHi)
An EXTR instruction is made up of two shifts, ORed together.
static bool isEquivalentMaskless(unsigned CC, unsigned width, ISD::LoadExtType ExtType, int AddConstant, int CompConstant)
static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG)
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
SDValue performSVESpliceCombine(SDNode *N, SelectionDAG &DAG)
static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT, unsigned Opcode, SelectionDAG &DAG)
static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N, SelectionDAG &DAG)
static bool isINSMask(ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits, APInt &UndefBits)
static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG)
static unsigned getIntrinsicID(const SDNode *N)
static bool IsSVECntIntrinsic(SDValue S)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool canEmitConjunction(const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool WillNegate, unsigned Depth=0)
Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
static bool isWideDUPMask(ArrayRef< int > M, EVT VT, unsigned BlockSize, unsigned &DupLaneOp)
Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vect...
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St, SDValue SplatVal, unsigned NumVecElts)
static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG, bool isSigned)
static SDValue performSetccMergeZeroCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG)
static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *ST)
static SDValue performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue removeRedundantInsertVectorElt(SDNode *N)
static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static SDValue legalizeSVEGatherPrefetchOffsVec(SDNode *N, SelectionDAG &DAG)
Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked ...
static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG)
static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc)
static bool hasPairwiseAdd(unsigned Opcode, EVT VT, bool FullFP16)
static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG)
static SDValue lowerConvertToSVBool(SDValue Op, SelectionDAG &DAG)
static SDValue performVectorShiftCombine(SDNode *N, const AArch64TargetLowering &TLI, TargetLowering::DAGCombinerInfo &DCI)
Optimize a vector shift instruction and its operand if shifted out bits are not used.
static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue combineSVEPrefetchVecBaseImmOff(SDNode *N, SelectionDAG &DAG, unsigned ScalarSizeInBytes)
Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses ...
static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St)
Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
unsigned getSignExtendedGatherOpcode(unsigned Opcode)
static SDValue performAddSubCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC)
changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true)
void selectGatherScatterAddrMode(SDValue &BasePtr, SDValue &Index, EVT MemVT, unsigned &Opcode, bool IsGather, SelectionDAG &DAG)
static SDValue combineSVEReductionFP(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc)
static unsigned getCmpOperandFoldingProfit(SDValue Op)
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static SDValue performConcatVectorsCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG)
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT, int Pattern)
static bool isEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseEXT, unsigned &Imm)
static bool isAllActivePredicate(SDValue N)
static SDValue tryCombineFixedPointConvert(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue performMulVectorExtendCombine(SDNode *Mul, SelectionDAG &DAG)
Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZE...
static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG)
static SDValue performVectorTruncateCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
static EVT getPromotedVTForPredicate(EVT VT)
static void changeFPCCToANDAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Convert a DAG fp condition code to an AArch64 CC.
static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X,...
static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG)
static bool isAllConstantBuildVector(const SDValue &PotentialBVec, uint64_t &ConstVal)
static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG)
static Value * UseTlsOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG)
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 regis...
static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG)
static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, AArch64CC::CondCode Cond)
static bool isSetCCOrZExtSetCC(const SDValue &Op, SetCCInfoAndKind &Info)
cl::opt< bool > EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))
static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG)
static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performPostLD1Combine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, bool IsLaneOp)
Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2)
Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
std::pair< SDValue, uint64_t > lookThroughSignExtension(SDValue Val)
static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG)
static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static PredicateConstraint parsePredicateConstraint(StringRef Constraint)
static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isPackedVectorType(EVT VT, SelectionDAG &DAG)
Returns true if VT's elements occupy the lowest bit positions of its associated register class withou...
static SDValue performAddSubLongCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static bool isTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v,...
static bool isZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue performSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static SDValue performExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert, SelectionDAG &DAG)
static SDValue tryCombineToBSL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl, SelectionDAG &DAG, SDValue Chain, bool IsSignaling)
static bool isSignExtended(SDNode *N, SelectionDAG &DAG)
static SDValue performUADDVCombine(SDNode *N, SelectionDAG &DAG)
bool getGatherScatterIndexIsExtended(SDValue Index)
static bool isZeroExtended(SDNode *N, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true)
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue combineSVEReductionInt(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static bool isCMN(SDValue Op, ISD::CondCode CC)
static SDValue tryCombineToEXTR(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low...
static bool isOperandOfVmullHighP64(Value *Op)
Check if Op could be used with vmull_high_p64 intrinsic.
static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode, SDValue Operand, SelectionDAG &DAG, int &ExtraSteps)
static bool isEssentiallyExtractHighSubvector(SDValue N)
static bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
static unsigned getExtFactor(SDValue &V)
getExtFactor - Determine the adjustment factor for the position when generating an "extract from vect...
static SDValue performExtractVectorEltCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performInsertVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG)
static const MVT MVT_CC
Value type used for condition codes.
static SDValue performCommonVectorExtendCombine(SDValue VectorShuffle, SelectionDAG &DAG)
Combines a dup(sext/zext) node pattern into sext/zext(dup) making use of the vector SExt/ZExt rather ...
static SDValue performAddDotCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performTBZCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
static SDValue foldTruncStoreOfExt(SelectionDAG &DAG, SDNode *N)
static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AArch64cc, SelectionDAG &DAG, const SDLoc &dl)
static bool performTBISimplification(SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
Simplify Addr given that the top byte of it is ignored by HW during address translation.
static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static cl::opt< bool > EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true))
static bool isZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v,...
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static std::pair< SDValue, SDValue > getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG)
#define FALKOR_STRIDED_ACCESS_MD
static const unsigned PerfectShuffleTable[6561+1]
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static bool isConstant(const MachineInstr &MI)
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static const MCPhysReg GPRArgRegs[]
Function Alias Analysis Results
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
PropagateLiveness Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
This file defines ARC utility functions which are used by various parts of the compiler.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
StandardInstrumentations SI(Debug, VerifyEach)
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
#define STATISTIC(VARNAME, DESC)
static const int BlockSize
This defines the Use class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static constexpr int Concat[]
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
unsigned getVarArgsFPRSize() const
void setVarArgsStackIndex(int Index)
void setTailCallReservedStack(unsigned bytes)
SmallVectorImpl< ForwardedRegister > & getForwardedMustTailRegParms()
void setIsSplitCSR(bool s)
int getVarArgsFPRIndex() const
void incNumLocalDynamicTLSAccesses()
void setBytesInStackArgArea(unsigned bytes)
int getVarArgsStackIndex() const
void setVarArgsGPRIndex(int Index)
int getVarArgsGPRIndex() const
void setHasSwiftAsyncContext(bool HasContext)
void setVarArgsFPRSize(unsigned Size)
unsigned getVarArgsGPRSize() const
unsigned getSRetReturnReg() const
void setSRetReturnReg(unsigned Reg)
unsigned getBytesInStackArgArea() const
void setJumpTableEntryInfo(int Idx, unsigned Size, MCSymbol *PCRelSym)
void setVarArgsFPRIndex(int Index)
void setVarArgsGPRSize(unsigned Size)
void setArgumentStackToRestore(unsigned bytes)
void UpdateCustomCalleeSavedRegs(MachineFunction &MF) const
static bool hasSVEArgsOrReturn(const MachineFunction *MF)
bool isTargetWindows() const
bool hasFuseLiterals() const
unsigned getPrefLoopLogAlignment() const
const AArch64RegisterInfo * getRegisterInfo() const override
unsigned getPrefFunctionLogAlignment() const
bool isMisaligned128StoreSlow() const
const AArch64InstrInfo * getInstrInfo() const override
unsigned getMaximumJumpTableSize() const
bool isTargetDarwin() const
bool isTargetILP32() const
ARMProcFamilyEnum getProcFamily() const
Returns ARM processor family.
unsigned classifyGlobalFunctionReference(const GlobalValue *GV, const TargetMachine &TM) const
bool isTargetMachO() const
bool supportsAddressTopByteIgnored() const
CPU has TBI (top byte of addresses is ignored during HW address translation) and OS enables it.
bool isTargetAndroid() const
const Triple & getTargetTriple() const
bool isCallingConvWin64(CallingConv::ID CC) const
bool outlineAtomics() const
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
bool isLittleEndian() const
bool hasAggressiveFMA() const
bool isXRegisterReserved(size_t i) const
bool requiresStrictAlign() const
bool isTargetFuchsia() const
bool predictableSelectIsExpensive() const
bool useSVEForFixedLengthVectors() const
unsigned getMinSVEVectorSizeInBits() const
bool hasCustomCallingConv() const
bool isTruncateFree(Type *Ty1, Type *Ty2) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
void initializeSplitCSR(MachineBasicBlock *Entry) const override
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
EVT getPromotedVTForPredicate(EVT VT) const
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
unsigned getVaListSizeInBits(const DataLayout &DL) const override
Returns the size of the platform's va_list object.
void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const override
Insert explicit copies in entry and exit blocks.
bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns true if the given (atomic) store should be expanded by the IR-level AtomicExpand pass into an...
bool generateFMAsInMachineCombiner(EVT VT, CodeGenOpt::Level OptLevel) const override
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, bool *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
Provide custom lowering hooks for some operations.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC) const
Selects the correct CCAssignFn for a given CallingConvention value.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ISD::SETCC ValueType.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
Selects the correct CCAssignFn for a given CallingConvention value.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
bool isLegalICmpImmediate(int64_t) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a ldN intrinsic.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
bool fallBackToDAGISel(const Instruction &Inst) const override
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isLegalInterleavedAccessType(VectorType *VecTy, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
For some targets, an LLVM struct type must be broken down into multiple simple types,...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
MachineBasicBlock * EmitLoweredCatchRet(MachineInstr &MI, MachineBasicBlock *BB) const
bool isZExtFree(Type *Ty1, Type *Ty2) const override
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const override
SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const override
If the target has a standard location for the unsafe stack pointer, returns the address of that locat...
InstructionCost getScalingFactorCost(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS) const override
Return the cost of the scaling factor used in the addressing mode represented by AM for this target,...
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isProfitableToHoist(Instruction *I) const override
Check if it is profitable to hoist instruction in then/else to if.
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const override
Return true if it is profitable to reduce a load to a smaller type.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a stN intrinsic.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
MachineBasicBlock * EmitF128CSEL(MachineInstr &MI, MachineBasicBlock *BB) const
LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &FuncAttributes) const override
LLT returning variant.
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool needsFixedCatchObjects() const override
Used for exception handling on Win64.
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
AArch64TargetLowering(const TargetMachine &TM, const AArch64Subtarget &STI)
bool isLegalAddImmediate(int64_t) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool shouldConsiderGEPOffsetSplit() const override
const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const override
Returns a 0 terminated array of registers that can be safely used as scratch registers.
bool isAllActivePredicate(SDValue N) const
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Returns false if N is a bit extraction pattern of (X >> C) & Mask.
bool enableAggressiveFMAFusion(EVT VT) const override
Enable aggressive FMA fusion on targets that want it.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override
EVT is not used in-tree, but is used by out-of-tree target.
bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
bool mergeStoresAfterLegalization(EVT VT) const override
SVE code generation for fixed length vectors does not custom lower BUILD_VECTOR.
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
unsigned logBase2() const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
int64_t getSExtValue() const
Get sign extended value.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
bool isFloatingPointOperation() const
BinOp getOperation() const
This is an SDNode representing atomic operations.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Equivalent to hasAttribute(AttributeList::FunctionIndex, Kind) but may be faster.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
const BlockAddress * getBlockAddress() const
A "pseudo-class" with methods for operating on BUILD_VECTORs.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
unsigned getNextStackOffset() const
getNextStackOffset - Return the next stack offset such that all stack slots satisfy their alignment r...
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
CCValAssign - Represent assignment of one arg/retval to a location.
Value * getArgOperand(unsigned i) const
unsigned getNumArgOperands() const
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
const APInt & getValue() const
Return the constant as an APInt value reference.
uint64_t getZExtValue() const
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
const Function & getFunction() const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
Type * getValueType() const
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
BasicBlock * GetInsertBlock() const
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This instruction inserts a single (scalar) element into a VectorType value.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
static LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
This is an important class for using LLVM in a threaded context.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
static ElementCount getScalable(ScalarTy MinVal)
LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
static ElementCount getFixed(ScalarTy MinVal)
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
This class is used to represent ISD::LOAD nodes.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
static MVT getFloatingPointVT(unsigned BitWidth)
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static auto integer_fixedlen_vector_valuetypes()
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static MVT getVectorVT(MVT VT, unsigned NumElements)
static auto fp_scalable_vector_valuetypes()
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setStackID(int ObjectIdx, uint8_t ID)
void setHasTailCall(bool V=true)
bool hasMustTailInVarArgFunc() const
Returns true if the function is variadic and contains a musttail call.
void setReturnAddressIsTaken(bool s)
void computeMaxCallFrameSize(const MachineFunction &MF)
Computes the maximum size of a callframe and the AdjustsStack property.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getBasePtr() const
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
unsigned getAlignment() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Class to represent pointers.
Type * getElementType() const
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const TargetSubtargetInfo & getSubtarget() const
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=None, int Offset=0, unsigned TargetFlags=0)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This instruction constructs a fixed permutation of two input vectors.
static bool isReverseMask(ArrayRef< int > Mask)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
VectorType * getType() const
Overload to return most specific vector type.
bool isZeroEltSplat() const
Return true if all elements of this shuffle are the same value as the first element of exactly one so...
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isExtractSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &Index)
Return true if this shuffle mask is an extract subvector mask.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
StackOffset is a class to represent an offset with 2 dimensions, named fixed and scalable,...
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
StringRef - Represent a constant reference to a string, i.e.
std::enable_if_t< std::numeric_limits< T >::is_signed, bool > getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
LLVM_NODISCARD StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
LLVM_NODISCARD size_t size() const
size - Get the string size.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setTargetDAGCombine(ISD::NodeType NT)
Targets should invoke this method for each target independent node that they want to provide a custom...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const
Returns the target-specific address of the unsafe stack pointer.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const
Check whether or not MI needs to be moved close to its uses.
void setMaximumJumpTableSize(unsigned)
Indicate the maximum number of entries in jump tables.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
unsigned getMaximumJumpTableSize() const
Return upper limit for number of entries in a jump table.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
void setCondCodeAction(ISD::CondCode CC, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool EnableExtLdPromotion
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned TLSSize
Bit size of immediate TLS offsets (0 == use the default).
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Triple - Helper class for working with autoconf configuration names.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
ScalarTy getFixedSize() const
ScalarTy getKnownMinSize() const
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt64Ty(LLVMContext &C)
bool isPointerTy() const
True if this is an instance of PointerType.
@ FloatTyID
32-bit floating point type
@ DoubleTyID
64-bit floating point type
static Type * getVoidTy(LLVMContext &C)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
ScalarTy getValue() const
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
Base class of all SIMD vector types.
Type * getElementType() const
Implementation for an ilist node.
ilist_node_impl()=default
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
static unsigned getNZCVToSatisfyCondCode(CondCode Code)
Given a condition code, return NZCV flags that would satisfy that condition.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_HI12
MO_HI12 - This flag indicates that a symbol operand represents the bits 13-24 of a 64-bit address,...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
@ GLDFF1S_SXTW_MERGE_ZERO
@ GLDFF1_SCALED_MERGE_ZERO
@ GLD1_SXTW_SCALED_MERGE_ZERO
@ FP_EXTEND_MERGE_PASSTHRU
@ FP_ROUND_MERGE_PASSTHRU
@ GLDFF1_SXTW_SCALED_MERGE_ZERO
@ UINT_TO_FP_MERGE_PASSTHRU
@ FROUNDEVEN_MERGE_PASSTHRU
@ GLD1S_UXTW_SCALED_MERGE_ZERO
@ GLDNT1_INDEX_MERGE_ZERO
@ GLDFF1_UXTW_SCALED_MERGE_ZERO
@ FNEARBYINT_MERGE_PASSTHRU
@ GLDFF1S_SCALED_MERGE_ZERO
@ GLDFF1S_UXTW_SCALED_MERGE_ZERO
@ ZERO_EXTEND_INREG_MERGE_PASSTHRU
@ NVCAST
Natural vector cast.
@ BITREVERSE_MERGE_PASSTHRU
@ GLDFF1S_UXTW_MERGE_ZERO
@ SIGN_EXTEND_INREG_MERGE_PASSTHRU
@ GLDFF1S_SXTW_SCALED_MERGE_ZERO
@ GLD1S_SCALED_MERGE_ZERO
@ SINT_TO_FP_MERGE_PASSTHRU
@ GLD1_UXTW_SCALED_MERGE_ZERO
@ GLD1S_SXTW_SCALED_MERGE_ZERO
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint8_t encodeAdvSIMDModImmType2(uint64_t Imm)
static bool isAdvSIMDModImmType9(uint64_t Imm)
static bool isAdvSIMDModImmType4(uint64_t Imm)
static bool isAdvSIMDModImmType5(uint64_t Imm)
static int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
static uint8_t encodeAdvSIMDModImmType7(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType12(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType10(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType9(uint64_t Imm)
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static bool isAdvSIMDModImmType7(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType5(uint64_t Imm)
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static bool isAdvSIMDModImmType10(uint64_t Imm)
static int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
static uint8_t encodeAdvSIMDModImmType8(uint64_t Imm)
static bool isAdvSIMDModImmType12(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType11(uint64_t Imm)
static bool isAdvSIMDModImmType11(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType6(uint64_t Imm)
static bool isAdvSIMDModImmType8(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType4(uint64_t Imm)
static bool isAdvSIMDModImmType6(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType1(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType3(uint64_t Imm)
static bool isAdvSIMDModImmType2(uint64_t Imm)
static bool isAdvSIMDModImmType3(uint64_t Imm)
static bool isAdvSIMDModImmType1(uint64_t Imm)
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
const unsigned RoundingBitsPos
static constexpr unsigned SVEBitsPerBlock
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ AArch64_SVE_VectorCall
Calling convention between AArch64 SVE functions.
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
@ Tail
Tail - This calling convention attemps to make calls as fast as possible while guaranteeing that tail...
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ SwiftTail
SwiftTail - This follows the Swift calling convention in how arguments are passed but guarantees tail...
@ C
C - The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ FLT_ROUNDS_
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isOverflowIntrOpRes(SDValue Op)
Returns true if the specified value is the overflow result from one of the overflow intrinsic nodes.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
bool match(Val *V, const Pattern &P)
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
match_combine_or< CastClass_match< OpTy, Instruction::ZExt >, CastClass_match< OpTy, Instruction::SExt > > m_ZExtOrSExt(const OpTy &Op)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
initializer< Ty > init(const Ty &Val)
CodeModel::Model getCodeModel()
bool hasAttachedCallOpBundle(const CallBase *CB)
---------------------— PointerInfo ------------------------------------—
bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool operator==(uint64_t V1, const APInt &V2)
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool RetCC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_AArch64_DarwinPCS_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
testing::Matcher< const detail::ErrorHolder & > Failed()
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool CC_AArch64_Win64_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool CC_AArch64_Win64_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool CC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Z
zlib style complession
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
bool is_splat(R &&Range)
Wrapper function around std::equal to detect if all elements in a container are same.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool CC_AArch64_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
@ Invalid
Denotes invalid value.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
bool isAcquireOrStronger(AtomicOrdering AO)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr unsigned BitWidth
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
gep_type_iterator gep_type_begin(const User *GEP)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
bool CC_AArch64_DarwinPCS_ILP32_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
static const MachineMemOperand::Flags MOStridedAccess
bool CC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Helper structure to keep track of a SET_CC lowered into AArch64 code.
Helper structure to keep track of ISD::SET_CC operands.
Helper structure to be able to read SetCC information.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Represent subnormal handling kind for floating point instruction inputs and outputs.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
EVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
bool isFixedLengthVector() const
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Describes a register that needs to be forwarded from the prologue to a musttail call.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static KnownBits commonBits(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits common to LHS and RHS.
unsigned getBitWidth() const
Get the bit width of this value.
Structure used to represent pair of argument number after call lowering and register used to transfer...
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Constraint for a predicate of the form "cmp Pred Op, OtherOp", where Op is the value the constraint a...
These are IR-level optimization flags that may be propagated to SDNodes.
void setAllowReassociation(bool b)
void setNoUnsignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
This structure contains all information that is necessary for lowering calls.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
Helper structure to keep track of SetCC information.